#! /usr/bin/env python3
# coding=utf-8
import asyncio
import random
import re
import time
from datetime import datetime

import httpx
from loguru import logger
from nonebot.adapters.cqhttp import MessageSegment, Message, Bot, Event
from tinydb import Query

from .config import hso_config
from .model import group_config, friend_config, Power, status, db_tmp, history

# -------------------------------------------------------
#                       Setu 类包装
# -------------------------------------------------------

Q = Query()

class Setu:
    def __init__(self, bot: Bot, event: Event, state: dict, **requests_kwargs):
        self.bot = bot
        self.event = event
        self.state = state
        self._REQUESTS_KWARGS = requests_kwargs
        # -----------------------------------
        self.config = hso_config  # 全局设置
        self.message = event.dict()
        self.type = self.message["message_type"]
        self.setu_level = 1  # 默认涩图等级
        self.current_config = None  # 当前配置
        self.power = Power()
        self.del_list = list()
        self.ess_list = list()
        self.r18 = None
        self.num = None
        self.tag = None
        self.api_set()

    def api_set(self):  # 开启api按钮
        priority = hso_config.priority
        for i in priority:
            setattr(self, f'api{i}', True)

    def update_status(self, get_num):
        date = datetime.now().strftime("%Y-%m-%d")
        if self.type == "private":
            mold = "user_id"
        elif self.type == "group":
            mold = "group_id"
        info = status.search((Q[mold] == self.message[mold]) & (Q["date"] == date))[0]
        info["num"] += get_num
        status.update(info, (Q[mold] == self.message[mold]) & (Q["date"] == date))

    def build_text(self, id=None, msg: str = None):
        if self.type == "group" and self.current_config[self.type]["at"]:
            at = True
        else:
            at = False
        text = {"id": id["message_id"], "msg": msg, "at": at}
        info = history.search(Q["msg"] == msg)
        if info:
            history.update({"id": id["message_id"]}, Q["msg"] == msg)
        else:
            history.insert(text)

    async def get_text(self, message_id):
        info = history.search(Q["id"] == message_id)[0]
        await self.send(msg=info["msg"], at=info["at"])

    @staticmethod
    def ifSent(pixiv_id, id):
        data = db_tmp.table("sentlist").search((Q["pixiv_id"] == pixiv_id) & (Q["id"] == id))
        if data:  # 如果有数据
            if time.time() - data[0]["time"] <= 36000:  # 发送过
                logger.info("id:{}发送过~".format(pixiv_id))
                return True
            else:
                db_tmp.table("sentlist").update({"time": time.time()}, (Q["pixiv_id"] == pixiv_id) & (Q["id"] == id))
                return False
        else:  # 没数据
            db_tmp.table("sentlist").insert({"pixiv_id": pixiv_id, "id": id, "time": time.time()})
            return False

    async def send(self, file: str = "", msg=None, at: bool = False):  # 发送图片或文字
        # if file[1:4] != "http" and file[1:4] != "":  # 本地文件
        #    file = "file:///" + file
        message_segment = list()
        if file:
            message_segment.append(MessageSegment.image(file=file))
        if msg:
            message_segment.append(MessageSegment.text(msg))
        if at:
            message_segment.append(MessageSegment.at(self.event.dict()["user_id"]))
        message = Message(message_segment)
        return await self.bot.send(event=self.event, message=message)

    async def withdraw(self, id=None):
        await asyncio.sleep(30)
        if id:
            return await self.bot.delete_msg(message_id=id)
        for x in self.del_list:
            await self.bot.delete_msg(message_id=x)

    @staticmethod
    async def build_msg(api="",
                        title="",
                        author="",
                        uid="",
                        author_id="",
                        url="",
                        url_original=""):  # 构建消息
        if api == "loli":  # lolicon.app
            msg = "标题:{title}\r\n作者:{author}\r\n[www.pixiv.net/users/{author_id}]\r\n作品id:{id}\r\n" \
                  "[www.pixiv.net/artworks/{id}]\r\n原图:{url_original}\r\n".format(title=title, id=uid, url=url,
                                                                                author_id=author_id, author=author,
                                                                                url_original=url_original)
        elif api == "y":  # yande.re
            msg = "标题:{title}\r\n作者:{author}\r\n原图:{url_original}\r\n(需要科学上网)\r\n".format(
                title=title,
                author=author,
                url_original=url_original
            )
        else:
            msg = "msg配置错误,请联系管理员"
        return msg

    async def sent(self, msg, url_large=None, url_original=None):
        if self.current_config[self.type]["original"]:  # 是否发送原图
            id = await self.send(file=url_original)
        else:
            id = await self.send(file=url_large)
        if self.type == "group" and "R-18" not in msg:
            self.ess_list.append(id["message_id"])  # 加精表单
        self.del_list.append(id["message_id"])  # 撤回表单
        self.build_text(id, msg)  # 记录下载地址

    async def api_0(self):  # 本地图片(需要启用mongodb)
        pass

    #  这是api书写的范例
    async def api_1(self):  # https://github.com/yuban10703 请不要过多调用造成yuban的困扰
        if self.api1 in vars() or self.num < 1:  # 判断是否开启
            return
        get_num = 0
        tag = ""
        url = "http://api.yuban10703.xyz:2333/setu_v4"
        if self.tag:
            tag = self.tag
        params = {"level": self.setu_level,
                  "num": self.num,
                  "tag": tag}
        try:
            async with httpx.AsyncClient() as client:
                res = await client.get(url, params=params, timeout=5)
                setu_data = res.json()
        except Exception as e:
            logger.warning("api1 boom~ :{}".format(e))
        else:
            if res.status_code == 200:
                for data in setu_data["data"]:
                    if self.ifSent(data["artwork"], self.event.get_user_id()):  # 判断是否发送过
                        continue
                    url_original = data["original"].replace("i.pximg.net", "i.pixiv.cat")  # 原图链接
                    url_large = data["large"].replace("i.pximg.net", "i.pixiv.cat")  # 高清链接
                    msg = await self.build_msg(api="loli", title=data["title"], author=data["author"],
                                               uid=data["artwork"],
                                               author_id=data["artist"], url_original=url_original)  # 组装消息
                    msg += "标签:{}\r\n".format(",".join(data["tags"]))
                    await self.sent(msg, url_original=url_original, url_large=url_large)  # 发送消息
                    get_num += 1
                    self.num -= 1

            # 打印获取到多少条
            self.update_status(get_num)  # 更新调用记录
            logger.info("从yubanのapi获取到{}张关于{}的setu  实际发送{}张".format(setu_data["count"], self.tag, get_num))

    async def api_2(self):  # https://api.lolicon.app/
        if not self.api2 in vars() or self.num < 1:  # 判断是否开启
            return
        if self.setu_level == 1:
            r18 = 0
        elif self.setu_level == 3:
            r18 = random.choice([0, 1])
        elif self.setu_level == 2:
            r18 = 1
        else:
            r18 = 0
        get_num = 0
        url = "https://api.lolicon.app/setu"
        params = {"r18": r18,
                  "apikey": self.config.lolicon_key,
                  "num": self.num,
                  "size1200": not bool(self.current_config[self.type]["original"])}
        if self.num > 10:
            params["num"] = 10
        if len(self.tag) != 1 or (len(self.tag[0]) != 0 and not self.tag[0].isspace()):  # 如果tag不为空(字符串字数不为零且不为空)
            params["keyword"] = self.tag
        async with httpx.AsyncClient() as client:
            res = await client.get(url, params=params, timeout=5)
            setu_data = res.json()
            if res.status_code == 200:
                for data in setu_data["data"]:
                    if self.ifSent(data["pid"], self.event.get_user_id()):  # 判断是否发送过
                        continue
                    msg = await self.build_msg(api="loli", title=data["title"], uid=data["pid"], author=data["author"],
                                               author_id=data["uid"],
                                               url_original="https://i.pixiv.cat/img-original/img/{}".format(
                                                   re.findall("img/(.*)", data["url"])[0].replace("_master1200", "")))
                    await self.sent(msg, url_large=data["url"])  # 发送消息
                    get_num += 1
                    self.num -= 1
                logger.info(
                    "从loliconのapi获取到{}张关于{}的Setu  实际发送{}张".format(setu_data["count"], self.tag, get_num))  # 打印获取到多少条
                self.update_status(get_num)  # 更新调用记录
            else:
                logger.warning("api2:{}".format(res.status_code))

    """
    def api_2(self):  # https://yande.re/ 需要梯子且速度极其慢，不建议使用,异步用法未写
        url = "https://yande.re/post.json"
        if config["proxies"]:
            _REQUESTS_KWARGS = {
                "proxies": {
                    "https": config["proxy"],  # "http://127.0.0.1:10809"  代理
                }
            }
        else:
            _REQUESTS_KWARGS = dict()
        if len(self.tag) > 0:
            tag_switch = 1
        else:
            tag_switch = 0
        params = {"api_version": 2,
                  "tags": self.tag,
                  "limit": self.num,
                  "include_tags": tag_switch,
                  "filter": 1}
        if self.num > 10:  # api限制不能大于10
            params["num"] = 10
        try:
            res = requests.get(url, params, **_REQUESTS_KWARGS)
            setu_data = res.json()
        except Exception as e:
            logger.error("api0 boom~")
            logger.error(e)
        else:
            if res.status_code == 200:
                for data in setu_data["posts"]:
                    id = data["id"]
                    file_url = data["file_url"]
                    if self.if_sent(id):  # 判断是否发送过
                        continue
                    url_original = data["source"]
                    msg = self.build_msg(level="api0", title=data["tags"], author=data["author"],
                                         url_original=url_original)
                    with requests.get(file_url, **_REQUESTS_KWARGS) as resp:
                        with open("./tmp.jpg", "wb") as fd:
                            fd.write(resp.content)
                    sendMsg.send_pic(self.ctx, msg, picPath="./tmp.jpg")
                    self.api_0_realnum += 1
                # else:
                #     logger.warning("api0:{}".format(res.status_code))
            logger.info(
                "从yandeのapi获取到{}张setu  实际发送{}张".format(len(setu_data["posts"]), self.api_0_realnum))  # 打印获取到多少条
    """

    async def processing_and_inspect(self):  # 处理消息+调用
        # -----------------------------------------------
        if self.num == "一" or self.num == "":
            self.num = 1
        elif self.num == "二" or self.num == "俩" or self.num == "两":
            self.num = 2
        elif self.num == "三":
            self.num = 3
        elif self.num != "":
            # 如果指定了数量
            try:
                self.num = int(self.num)
            except ValueError:  # 出错就说明不是数字
                await self.send(msg="不会真的有人连数数字都不会输入吧！")
                return
            if self.num <= 0:  # ?????
                await self.send(msg="你想笑死我好继承我的负产吗¿¿¿")
                return
        else:  # 未指定默认1
            self.num = 1
        # -----------------------------------------------
        if not self.current_config[self.type]["setu"]:  # setu开关判断
            return await self.send(msg="啊嘞啊嘞，涩图还没开呢~")
        if self.num > self.current_config[self.type]["max_num"]:  # 最大数量判断
            return await self.send(msg="要这么多涩图你怎么不冲死呢¿")
        if self.r18:
            self.setu_level = self.current_config[self.type]["setu_level"]  # r18判断
            if self.setu_level < 2:
                await self.send(msg="太涩了，你不能看哼~")
                return
        if self.type == "private":  # 优先级
            if self.r18:
                self.setu_level = 2
            logger.info(self.current_config)
            if self.current_config[self.type]["setu_level"] is dir():
                self.setu_level = self.current_config[self.type]["setu_level"]
            date = datetime.now().strftime("%Y-%m-%d")
            info = status.search((Q["user_id"] == self.message["user_id"]) & (Q["date"] == date))
            if info:
                info = info[0]
                info["num"] += self.num
                status.update(info, (Q["user_id"] == self.message["user_id"]) & (Q["date"] == date))
            else:
                info = {
                    "user_id": self.message["user_id"],
                    "num": 0,
                    "date": date
                }
                status.insert(info)
        elif self.type in ["group", "temp"]:
            date = datetime.now().strftime("%Y-%m-%d")
            if self.type == "group":  # 统计调用数量
                info = status.search((Q["group_id"] == self.message["group_id"]) & (Q["date"] == date))
                if info:
                    info = info[0]
                else:
                    info = {
                        "group_id": self.message["group_id"],
                        "num": 0,
                        "date": date
                    }
                    status.insert(info)
                    info = status.search((Q["group_id"] == self.message["group_id"]) & (Q["date"] == date))[0]
                now_num = info["num"] + self.num
                if self.current_config["group"]["top"] >= now_num:  # 上限判断
                    self.call = await self.send(
                        msg="当天已调用：{}\r\n群剩余调用：{}".format(str(now_num),
                                                          str(self.current_config["group"]["top"] - now_num)))
                elif self.current_config["group"]["top"] == 0:
                    pass
                else:
                    self.call = await self.send(msg=f"当天已调用：{str(info['num'])}\r\n再调用{str(self.num)}超过上限了呢~")
        await self.action()

    async def main(self):  # 判断消息类型给对应函数处理
        self.r18 = self.state["_matched_groups"][2]
        self.num = self.state["_matched_groups"][0]
        self.tag: list = [i for i in list(set(re.split(r",|，|\.|-| |_|/|\\", self.state["_matched_groups"][1]))) if
                          i != ""]  # 分割tag+去重+去除空元素
        if self.type != "private":  # 群聊or临时会话
            data = group_config.search(Q["group_id"] == self.message["group_id"])[0]  # 数据库
            if data:  # 查询group数据库数据
                self.current_config = data
            else:
                await self.power.group_build(self.message["group_id"])
                self.current_config = group_config.search(Q["group_id"] == self.message["group_id"])[0]
            await self.processing_and_inspect()
        else:  # 好友会话
            if hso_config.friend:
                pass
            else:
                return await self.send(msg="全局设置中好友调用未开启")
            data = friend_config.search(Q[self.type]["user_id"] == self.message["user_id"])
            if data:  # 该QQ如果自定义过
                self.current_config = data[0]
            else:
                data = {"private": {
                    "user_id": self.message["user_id"],
                    "setu_level": 2,
                    "original": False,
                    "setu": True,
                    "r18": True,
                    "max_num": 3}}
                self.current_config = data
                friend_config.insert(data)
                logger.info(f"无用户{self.message['user_id']}数据，创建成功~")
                # 如果没有自定义 就是默认行为
                await self.processing_and_inspect()

    async def set_essence_msg(self, message_id):  # 添加精华消息
        await self.bot.call_api('set_essence_msg', message_id=message_id)

    async def action(self):  # 判断数量
        for i in self.config.priority:
            if getattr(self, f"api{i}"):
                t = eval(f"self.api_{i}")
                await t()
        if self.type == "group":
            if self.current_config["group"]["essence"]:
                for i in self.ess_list:
                    await self.set_essence_msg(i)
            if self.current_config["group"]["top"] != 0:
                await self.withdraw(id=self.call["message_id"])
            if self.current_config[self.type]["revoke"]:  # 撤回
                await self.withdraw()
        if self.num != 0:
            await self.send(msg="淦！你的xp好奇怪啊！")
            return
        return


class Solution(object):
    def isValidSerialization(self, preorder):
        p = preorder.split(',')
        slot = 1
        for node in p:
            if slot == 0:
                return False
            if node == '#':
                slot -= 1
            else:
                slot += 1

        return slot == 0

preorder = "9,3,4,#,#,1,#,#,2,#,6,#,#"
res = Solution().isValidSerialization(preorder)
print(res)


from sys import stdin,stdout

def countsetbits(n):
    if n==0:
        return 0
    else:
        return 1+ countsetbits(n&(n-1))

test=int(stdin.readline())
for t in range(test):
    n,q=stdin.readline().split()
    n,q=int(n),int(q)

    ls=[int(a) for a in stdin.readline().split()]

    for i in range(q):
        e,o=0,0
        p=int(stdin.readline())
        for j in range(n):
            temp=int(countsetbits(p^ls[j]))
            if temp%2==0:
                e+=1
            else:
                o+=1
        stdout.write(str(e)+" "+str(o)+"\n")


import Utils
from Position import Position
from Piece import Piece
from Action import Action

# this class implements the getPossibleActions for each type of piece

class Knight(Piece):

    # constructor
    def __init__(self, color):
        self.m_color = color
        self.coordinates = [(-2, -1), (-2, +1), (+2, -1), (+2, +1), (-1, -2), (-1, +2), (+1, -2), (+1, +2)]
        if color == 0:
            self.m_type = Utils.wKnight
        else:
            self.m_type = Utils.bKnight

    # this method must be completed with all the possible pieces
    def getPossibleActions(self, state):

        return self.getMovesByCoordinates(state)


#while 控制循环
#语法：
#while 条件表达式：#逻辑 成员 比较 空数据 布尔值
     #代码块
#执行规律：首先判断while后面的条件表达式是否成立
#如果为true，那就执行代码块，执行完毕后，继续判断
#如果为flase，不进入内部执行代码块
#防止代码进入死循环，加一个变量来控制循环次数
#利用break和continue进行控制循环

# #利用whlie循环，实现1-100的整数相加
# a = 1
# sum = 0
# while a <=100:
#     sum += a
#     a = a+1
# print("运行了{0}次，1-100的和为{1}".format(a,sum))

#break & continue
a = 1
sum = 0
while a>0:
    sum+=a
    a+=1
    if a > 100:
        break
    else:
        continue
print("运行了{0}次，1-100的和为{1}".format(a,sum))


# built-in
import asyncio
from pathlib import Path

# project
from dephell.repositories.local import LocalRepo

loop = asyncio.get_event_loop()

def test_root_file():
    repo = LocalRepo(Path() / 'tests' / 'requirements' / 'setup.py')
    root = repo.get_root(name='dephell', version='0.2.0')
    assert root.name == 'dephell'
    assert root.version == '0.2.0'
    assert root.description == 'Dependency resolution for Python'

def test_root_dir():
    repo = LocalRepo(Path() / 'tests' / 'requirements' / 'egg-info')
    root = repo.get_root(name='dephell', version='0.2.0')
    assert root.name == 'dephell'
    assert root.version == '0.2.1'
    assert root.links['home'] == 'https://github.com/orsinium/dephell'

def test_deps_file():
    repo = LocalRepo(Path() / 'tests' / 'requirements' / 'setup.py')
    coroutine = repo.get_dependencies(name='dephell', version='0.2.0')
    deps = loop.run_until_complete(asyncio.gather(coroutine))[0]
    deps = {dep.name: dep for dep in deps}
    assert set(deps) == {'attrs', 'cached-property', 'packaging', 'requests', 'colorama'}


import glob
import os
import subprocess

class FileNomination:
    def __init__(self, vim, options):
        self.command = []
        self.prune_paths = []
        self.remove_paths = []

        for path in options.get('ignore_paths', []):
            self.ignorePath(path)

    def ignorePath(self, path):
        if os.path.isdir(path):
            if path[-1] == '/':
                path = path[:-1]

            self.prune_paths.append('./' + path)
        elif os.path.isfile(path):
            self.remove_paths.append('./' + path)

    def getCommand(self, expression):
        if not self.command:
            self.command = ['find', '-L', '.']

            agignore_filename = os.path.expanduser('~/.agignore')

            if os.path.isfile(agignore_filename):
                agignore_file = open(agignore_filename, 'r')

                for line in agignore_file:
                    self.ignorePath(line)

            if os.path.isdir('.git'):
                for git_path in glob.glob('.git*'):
                    self.ignorePath(git_path)

                output = subprocess.check_output(['git', 'clean', '-ndX'])

                for line in output.split('\n'):
                    # Remove 'Would remove ' from beginning of line
                    self.ignorePath(line[13:])

            for prune_path in self.prune_paths:
                self.command.extend(['-path', prune_path, '-prune', '-o'])

            for remove_path in self.remove_paths:
                self.command.extend(['!', '-path', remove_path])

            self.command.extend(['-type', 'f', '-path', ' ', '-printf', '%P\n'])

        if expression == '':
            expression = '""'

        self.command[-3] = '%s' % expression

        return self.command

class SpotNomination:
    def __init__(self, vim, options):
        self.vim = vim

    def getCommand(self, expression):
        if expression == '*':
            expression = self.vim.eval('input("expression: ")')

        return ['ag', '--follow', '--nocolor', '--nogroup', '--column', expression]

election_nominations = {
        'file': FileNomination,
        'spot': SpotNomination,
        }


# Copyright 2016-2022 The FEAGI Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import logging
import os
import platform
import psutil
import string
import random
from queue import Queue
from tempfile import gettempdir
from threading import Thread
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from datetime import datetime
from collections import deque
from inf import runtime_data, disk_ops, settings
from configparser import ConfigParser
from shutil import copyfile
from evo.stats import voxel_dict_summary
from inf.messenger import Pub
from evo.neuroembryogenesis import generate_plasticity_dict
from evo.genome_processor import *

log = logging.getLogger(__name__)

# def init_hw_controller():
#     """
#     Loads the proper controller module based on the robot species defined within genome
#     """
#     import sys
#     print("controller path:", runtime_data.hw_controller_path)
#     sys.path.insert(1, runtime_data.hw_controller_path)
#     import controller

def detect_hardware():
    """
    Identifies the type of hardware the brain is running on so the right capabilities can be utilized
    """
    # todo

    try:
        with open('/sys/firmware/devicetree/base/model', "r") as file:
            if "Raspberry" in file.read():
                runtime_data.hardware = "raspberry_pi"
    except:
        print("Need to figure how other platforms can be detected")

    runtime_data.hw_controller_path = '../third_party/' + \
                                      runtime_data.genome['species']['brand'] + '/' +\
                                      runtime_data.genome['species']['model'] + '/controller.py'
    print("Hardware controller path: ", runtime_data.hw_controller_path)

def init_container_variables():
    """
    Identifies variables set by containers and sets them in FEAGI runtime parameters
    """

    if os.environ.get('CONTAINERIZED', False):
        runtime_data.running_in_container = True
    if os.environ.get('influxdb', False):
        runtime_data.influxdb = True
    if os.environ.get('mongodb', False):
        runtime_data.mongodb = True
    if os.environ.get('gazebo', False):
        runtime_data.gazebo = True

def running_in_container():
    """
    Identifies if FEAGI is running in a container or not based on the ENV variable set during the container creation

    Warning: This method of detection is not reliable as it will fail if during container formation ENV is not set

    """
    container_check = os.environ.get('CONTAINERIZED', False)

    if container_check:
        print("FEAGI is running in a Docker container")
    else:
        print("FEAGI is not running outside container")
    return container_check

def assess_max_thread_count():
    """
    FEAGI requires approxiately 1GB of memory per process. This function determines the proper number of max threads
    used by FEAGI by taking into consideration the number of CPU core count as well as available memory on the system.
    """

    cpu_core_count = psutil.cpu_count()
    print("Device CPU Core Count = ", cpu_core_count)

    free_mem = psutil.virtual_memory().available
    print("Device Free Memory = ", free_mem)

    max_thread_count = min(int(free_mem / 1024 ** 3), cpu_core_count)

    if max_thread_count == 0:
        max_thread_count = 1

    return max_thread_count

def run_id_gen(size=6, chars=string.ascii_uppercase + string.digits):
    """
    This function generates a unique id which will be associated with each neuron
    :param size:
    :param chars:
    :return:

    Rand gen source partially from:
    http://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
    """
    runtime_data.brain_run_id = \
        (str(datetime.now()).replace(' ', '_')).replace('.', '_').replace(':', '-')+'_'+(''.join(random.choice(chars)
                                                                               for _ in range(size)))+'_R'

def init_parameters(ini_path='./feagi_configuration.ini'):
    """To load all the key configuration parameters"""
    feagi_config = ConfigParser()
    feagi_config.read(ini_path)
    runtime_data.parameters = {s: dict(feagi_config.items(s)) for s in feagi_config.sections()}
    if not runtime_data.parameters["InitData"]["working_directory"]:
        runtime_data.parameters["InitData"]["working_directory"] = gettempdir()
    log.info("All parameters have been initialized.")

def init_working_directory():
    """
    Creates needed folder structure as the working directory for FEAGI. This includes a folder to house connectome
    as well as folders needed for IPU/OPU to ingress/egress data accordingly.

    """
    if platform.system() == 'Windows':
        runtime_data.working_directory = runtime_data.parameters["InitData"]["working_directory"] + '\\' + \
                                         runtime_data.brain_run_id

        # Create connectome directory if needed
        # todo: need to consolidate the connectome path as currently captured in two places
        runtime_data.connectome_path = runtime_data.working_directory + '\\connectome\\'
        runtime_data.parameters["InitData"]["connectome_path"] = runtime_data.connectome_path

        if not os.path.exists(runtime_data.connectome_path):
            os.makedirs(runtime_data.connectome_path)

        # Create IPU directories if needed
        # todo: figure best way to obtain the following list. possibly from genome
        directory_list = ['ipu', 'opu_vision', 'opu_utf', 'opu_auditory']
        for _ in directory_list:
            ipu_path = runtime_data.working_directory + '\\' + _
            if not os.path.exists(ipu_path):
                os.makedirs(ipu_path)
            runtime_data.paths[_] = runtime_data.working_directory + _
    else:
        runtime_data.working_directory = runtime_data.parameters["InitData"]["working_directory"] + '/' + \
                                         runtime_data.brain_run_id
        runtime_data.connectome_path = runtime_data.working_directory + '/connectome/'
        runtime_data.parameters["InitData"]["connectome_path"] = runtime_data.connectome_path

        if not os.path.exists(runtime_data.connectome_path):
            os.makedirs(runtime_data.connectome_path)
        # copyfile(runtime_data.parameters["InitData"]["static_genome_path"], runtime_data.connectome_path + '/')

        directory_list = ['ipu', 'opu_vision', 'opu_utf', 'opu_auditory']
        for _ in directory_list:
            ipu_path = runtime_data.working_directory + '/' + _
            if not os.path.exists(ipu_path):
                os.makedirs(ipu_path)
            runtime_data.paths[_] = runtime_data.working_directory + _
        # print(runtime_data.paths)

def init_genome():
    print("\nInitializing genome...\n")
    # The following stages the genome in the proper connectome path and loads it into the memory
    disk_ops.genome_handler(runtime_data.connectome_path)

    try:
        if runtime_data.genome['version'] == "2.0":
            print("\n\n\n************ Genome Version 2.0 has been detected **************\n\n\n")
            runtime_data.genome_ver = "2.0"
            runtime_data.cortical_list = genome_2_cortical_list(runtime_data.genome['blueprint'])
            genome2 = genome_2_1_convertor(flat_genome=runtime_data.genome['blueprint'])
            genome_2_hierarchifier(flat_genome=runtime_data.genome['blueprint'])
            runtime_data.genome['blueprint'] = genome2['blueprint']
    except KeyError as e:
        print("Error:", e)
        print("Genome version not available; assuming Genome 1.0 procedures.")
        runtime_data.cortical_list = genome_1_cortical_list(runtime_data.genome['blueprint'])
        pass

def init_genome_post_processes():
    """
    Augments genome with details that can improve the run-time performance by reducing frequent operations
    """
    # Augment cortical dimension dominance e.g. is it longer in x dimension or z
    for cortical_area in runtime_data.cortical_list:
        block_boundaries = runtime_data.genome["blueprint"][cortical_area]["neuron_params"]["block_boundaries"]
        # block_boundaries = runtime_data.genome["blueprint"][]
        dominance = block_boundaries.index(max(block_boundaries))
        runtime_data.genome['blueprint'][cortical_area]['dimension_dominance'] = dominance

def init_timeseries_db():
    """
    Conducts needed checks to ensure the time-series database is ready

    Utilizing InfluxDb as the time-series database
    """
    from inf import db_handler
    runtime_data.influxdb = db_handler.InfluxManagement()
    runtime_data.influxdb.test_influxdb()
    return

def init_genome_db():
    print("- Starting MongoDb initialization...")
    from inf import db_handler
    runtime_data.mongodb = db_handler.MongoManagement()
    runtime_data.mongodb.test_mongodb()
    print("+ MondoDb has been successfully initialized")
    return

def init_data_sources():
    """To validate and initialize all data sources and databases"""
    print("\nInitializing databases...")
    # Light mode is a switch to help globally turn off some of the resource intensive processes
    # todo: Light mode needs better definition as currently it is more db centric and not serving the key purpose
    if not runtime_data.parameters['Switches']['light_mode']:
        if runtime_data.parameters['Database']['mongodb_enabled']:
            init_genome_db()
        else:
            print("    MongoDb:", settings.Bcolors.RED + "Disabled" + settings.Bcolors.ENDC)

        if runtime_data.parameters['Database']['influxdb_enabled']:
            init_timeseries_db()
        else:
            print("    InfluxDb:", settings.Bcolors.RED + "Disabled" + settings.Bcolors.ENDC)

        log.info("All data sources have been initialized.")
    else:
        print("FEAGI is operating in LIGHT MODE where no database is utilized.")

    return

def init_resources():

    if runtime_data.parameters['System']['max_core']:
        print("Max thread count was overwritten to:", runtime_data.parameters['System']['max_core'])
    else:
        runtime_data.parameters['System']['max_core'] = assess_max_thread_count()
        print("Max thread count was set to ", runtime_data.parameters['System']['max_core'])

def initialize():
    runtime_data.last_alertness_trigger = datetime.now()
    run_id_gen()
    init_parameters()
    init_io_channels()
    init_working_directory()
    init_container_variables()
    init_data_sources()
    init_genome()
    detect_hardware()
    init_genome_post_processes()
    init_resources()
    generate_plasticity_dict()
    runtime_data.fcl_queue = Queue()

def init_burst_engine():
    print("\n\n")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("**** **** **** **** ****       Starting the burst_manager engine...        **** **** ****")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("**** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** ****")
    print("\n\n")

    print(runtime_data.parameters['Switches']['use_static_genome'])

    disk_ops.genome_handler(runtime_data.parameters['InitData']['connectome_path'])

    runtime_data.termination_flag = False

    runtime_data.burst_timer = float(runtime_data.parameters["Timers"]["burst_timer"])
    print("Burst time has been set to:", runtime_data.burst_timer)

    if runtime_data.parameters["Logs"]["print_voxel_dict_report"]:
        print("Block Dictionary Report:")
        voxel_dict_summary(runtime_data.voxel_dict, verbose=True)

def exit_burst_process():
    print(settings.Bcolors.YELLOW + '>>>Burst Exit criteria has been met!   <<<' + settings.Bcolors.ENDC)
    runtime_data.live_mode_status = 'idle'
    runtime_data.burst_count = 0
    runtime_data.parameters["Switches"]["ready_to_exit_burst"] = True
    runtime_data.parameters["Auto_injector"]["injector_status"] = False
    if runtime_data.parameters["Switches"]["capture_brain_activities"]:
        disk_ops.save_fcl_to_disk()

def init_io_channels():
    # Initialize ZMQ connections
    try:
        # opu_socket = 'tcp://0.0.0.0:' + runtime_data.parameters['Sockets']['feagi_outbound_port']
        # print("OPU socket is:", opu_socket)
        # runtime_data.opu_pub = Pub(opu_socket)
        # print("OPU channel as been successfully established at ",
        #       runtime_data.parameters['Sockets']['feagi_outbound_port'])

        if runtime_data.parameters['Switches']['zmq_activity_publisher']:
            runtime_data.brain_activity_pub = True
        #     brain_activities_socket = 'tcp://0.0.0.0:' + runtime_data.parameters['Sockets']['brain_activities_pub']
        #     print("Brain activity publisher socket is:", brain_activities_socket)
        #     runtime_data.brain_activity_pub = PubBrainActivities(brain_activities_socket)

        if running_in_container():
            print(">>>  >>>> >>> >>>> >>> >> > >> > >> > > >   Brain is running in a container")
            if runtime_data.parameters['Sockets']['feagi_inbound_port_godot']:
                runtime_data.router_address_godot = "tcp://" + runtime_data.parameters['Sockets']['godot_host_name'] + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_godot']
            if runtime_data.parameters['Sockets']['feagi_inbound_port_gazebo']:
                runtime_data.router_address_gazebo = "tcp://" + runtime_data.parameters['Sockets']['gazebo_host_name'] + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_gazebo']
            if runtime_data.parameters['Sockets']['feagi_inbound_port_virtual']:
                runtime_data.router_address_virtual = "tcp://" + runtime_data.parameters['Sockets']['virtual_host_name'] + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_virtual']
        else:
            print(">>>  >>>> >>> >>>> >>> >> > >> > >> > > >   Brain is NOT running in a container  ******* ** * * *")
            if runtime_data.parameters['Sockets']['feagi_inbound_port_godot']:
                runtime_data.router_address_godot = "tcp://127.0.0.1" + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_godot']
            if runtime_data.parameters['Sockets']['feagi_inbound_port_gazebo']:
                runtime_data.router_address_gazebo = "tcp://127.0.0.1" + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_gazebo']
            if runtime_data.parameters['Sockets']['feagi_inbound_port_virtual']:
                runtime_data.router_address_virtual = "tcp://127.0.0.1" + ':' + runtime_data.parameters['Sockets'][
                    'feagi_inbound_port_virtual']

        print("Router addresses has been set")
    except KeyError as e:
        print('ERROR: OPU socket is not properly defined as part of feagi_configuration.ini\n', e)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from xml.createXml import Xml
import os

class createFeaturesXml:

	def __init__(self, path, node, triple, oss_region, bb):
		Enodebrel = Xml().getEnodebRel()									### Indica la relaease del nodo
		self.OSS_REL = Xml().getOssRel()
		self.node = node
		self.oss_region = oss_region
		self.bb = bb
		self.path_FeaturesXml = os.path.join(os.path.join(path, node), '2_'+self.node+'_feature_'+Enodebrel+'.xml')
		if self.bb:
			self.FeaturesModel = open(os.path.join(path,'Model_Feature_BB5216_'+Enodebrel+'.xml'), 'r')
		else:
			self.FeaturesModel = open(os.path.join(path,'Model_Feature_DUS41_'+Enodebrel+'.xml'), 'r')

	def go(self):
		s = ''
		for line in self.FeaturesModel:
			if '$$$OSS_REGION$$$' in line:
				s += line.replace('$$$OSS_REGION$$$', self.oss_region)
			elif '<xn:MeContext id="' in line:
				s += '				<xn:MeContext id="'+self.node+'">\n'
			elif '<xn:ManagedElement id="' in line:
				if self.bb:
					s += '					<xn:ManagedElement id="'+self.node+'">\n'
				else:
					s += '					<xn:ManagedElement id="1">\n'
			elif '$$$OSS_REL$$$' in line:
				s += line.replace('$$$OSS_REL$$$', self.OSS_REL)
			else:
				s += line
		FeaturesXml = open(self.path_FeaturesXml,'w')
		FeaturesXml.write(s)
		FeaturesXml.close()


#!/usr/bin/env python
# **********************************************************************
#
# Copyright (c) 2003-2018 ZeroC, Inc. All rights reserved.
#
# **********************************************************************

import sys
import Ice
import IceStorm
from os import listdir
from os.path import isfile, join

Ice.loadSlice('../META/server.ice')
import mp3App
import vlc

class ServerI(mp3App.Function):

    playList = []
    instance = vlc.Instance()
    player = instance.media_player_new()

    def __init__(self):
        print("server1 init")
        self.receivePlaylist()

    def sendPlayList(self, seq, current=None):
        self.playList = seq

    def receivePlaylist(self, current=None):
        self.playList = [f for f in listdir("S1Playlist/") if isfile(join("S1Playlist/", f))]
        print("Musics on server : ")
        print(self.playList)
        return self.playList

    def add(self, son, current=None):
        if not self.searchByName(son.name):
            self.playList.append(son)
            print("'" + son.name + "' added to playList")

    def remove(self, name, current=None):
        for aSong in self.playList:
            if aSong.name == name:
                self.playList.remove(aSong)
                print("'" + aSong.name + "' removed from playList\n")

    def searchByName(self, name, current=None):
        for aSong in self.playList:
            if aSong.name == name:
                print("Music '" + aSong.name + "' found in playList \n")
                return True
        return False

    def searchByGenre(self, genre, current=None):
        print("Genre '" + genre + "':")
        for aSong in self.playList:
            if aSong.genre == genre:
                print("Music '" + aSong.name + "'")
        print("\n")

    def searchByArtist(self, artist, current=None):
        print("Artist '" + artist + "':")
        for aSong in self.playList:
            if aSong.artist == artist:
                print("Music '" + aSong.name + "'")
        print("\n")

    def printPlayList(self, current=None):
        print("PlayList: ")

        for aSong in self.playList:
            print(aSong.name + " || " + aSong.artist + " || " + aSong.genre + " || " + aSong.length + " || " + aSong.path)
        print("\n")

    def playMusic(self, music, current=None):
        if self.player is None and self.instance is None:
            self.instance = vlc.Instance()
            self.player = self.instance.media_player_new()

        if self.playList.__contains__(music):
            options = ':sout=#transcode{vcodec=none,acodec=mp3,ab=128,channels=2,samplerate=44100}:http{mux=mp3,dst=:8080/player.mp3}'
            media = self.instance.media_new('/S1Playlist/' + music, options)
            print("playing music : " + music)
            self.player.set_media(media)
            self.player.play()
        else:
            print("music not available on server")

    def stopMusic(self, current=None):
        if self.player is not None:
            print("stoping music")
            self.player.stop()
            self.player.release()
            self.instance = None
            self.player = None
            # http://localhost:8080

def run(communicator):

    batch = False
    option = "Twoway"
    topicName = "mp3App"
    id = "1"
    retryCount = "1"

    if batch and (option in ("Twoway", "Ordered")):
        print("batch can only be set with oneway or datagram")
        sys.exit(1)

    manager = IceStorm.TopicManagerPrx.checkedCast(communicator.propertyToProxy('TopicManager.Proxy'))
    if not manager:
        print(" invalid proxy")
        sys.exit(1)

    #
    # Retrieve the topic.
    #
    try:
        topic = manager.retrieve(topicName)
    except IceStorm.NoSuchTopic as e:
        try:
            topic = manager.create(topicName)
        except IceStorm.TopicExists as ex:
            print("temporary error. try again")
            sys.exit(1)

    adapter = communicator.createObjectAdapter("mp3App.Subscriber")

    #
    # Add a servant for the Ice object. If --id is used the identity
    # comes from the command line, otherwise a UUID is used.
    #
    # id is not directly altered since it is used below to detect
    # whether subscribeAndGetPublisher can raise AlreadySubscribed.
    #

    subId = Ice.Identity()
    subId.name = id
    if len(subId.name) == 0:
        subId.name = Ice.generateUUID()
    subscriber = adapter.add(ServerI(), subId)

    #
    # Activate the object adapter before subscribing.
    #
    adapter.activate()

    qos = {}
    if len(retryCount) > 0:
        qos["retryCount"] = retryCount

    try:
        topic.subscribeAndGetPublisher(qos, subscriber)
    except IceStorm.AlreadySubscribed as ex:
        # If we're manually setting the subscriber id ignore.
        if len(id) == 0:
            raise
        print("reactivating persistent subscriber")

    communicator.waitForShutdown()

    #
    # Unsubscribe all subscribed objects.
    #
    topic.unsubscribe(subscriber)

#
# Ice.initialize returns an initialized Ice communicator,
# the communicator is destroyed once it goes out of scope.
#
with Ice.initialize(sys.argv, "../META/config.sub") as communicator:
    status = run(communicator)


load("@bazel_skylib//lib:paths.bzl", "paths")

def _deploy_mysql_impl(ctx):
    template_name = paths.basename(ctx.file._script_template.short_path)
    script = ctx.actions.declare_file("rendered_{}".format(template_name))

    ctx.actions.expand_template(
        template = ctx.file._script_template,
        output = script,
        substitutions = {
            "{MYSQL_CHART}": ctx.file._mysql_chart.path,
            "{HELM}": ctx.executable._helm.short_path,
            "{KUBECTL}": ctx.executable._kubectl.short_path,
        },
        is_executable = True,
    )

    runfiles = [
        ctx.file._mysql_chart,
        ctx.executable._helm,
        ctx.executable._kubectl,
    ]
    return [DefaultInfo(
        executable = script,
        runfiles = ctx.runfiles(files = runfiles),
    )]

deploy_mysql = rule(
    implementation = _deploy_mysql_impl,
    attrs = {
        "_mysql_chart": attr.label(
            allow_single_file = True,
            default = "@mysql_chart//file",
        ),
        "_helm": attr.label(
            allow_single_file = True,
            cfg = "host",
            default = "@helm//:binary",
            executable = True,
        ),
        "_kubectl": attr.label(
            allow_single_file = True,
            cfg = "host",
            default = "@kubectl//:binary",
            executable = True,
        ),
        "_script_template": attr.label(
            allow_single_file = True,
            cfg = "host",
            default = "//dev/external_database:deploy_mysql.sh",
        ),
    },
    executable = True,
)


# Victor Ferreira Ferrari, RA 187890
# MC920 - Introduction to Image Digital Processing
# Project 5 - Principal Component Analysis
# Last modified: 21/11/2019

from cv2 import imwrite, imread
from cv2 import imshow, waitKey, destroyAllWindows, IMREAD_COLOR
from os.path import join, basename, getsize
import matplotlib.pyplot as plt
import numpy as np
import argparse

def main():
    parser = argparse.ArgumentParser(description='Uses Principal Component Analysis (PCA) for compression.')
    parser.add_argument('file', help='Name of the file containing the image (PNG)')
    parser.add_argument('comp', help='Amount of components to split.', type=int)
    parser.add_argument('--plot', help='Plot component variance graph.', action='store_true')
    parser.add_argument('--folder', help='Folder to save compressed image (defaults to Outputs/).', default='Outputs')
    args = parser.parse_args()

    # Read color image
    img = imread(args.file, IMREAD_COLOR)

    # Checks if plot mode.
    if args.plot:
        plot_components(img, args.comp)
        name = 'comp_graph_' + basename(args.file)
        plt.savefig(join(args.folder,name))
        plt.show()
    else:
        # Compress image
        out = pca_compression(img, args.comp)

        # Show/Save result and original image
        base = basename(args.file)
        name = str(args.comp) + 'comp_' + base
        show_image(out.astype(np.uint8), name='Compressed')
        save_image(out, name, args.folder)
        save_image(img, base, args.folder)

        # Evaluate result
        metrics = eval_compression(img.astype(np.float64), out, join(args.folder, base), join(args.folder, name))
        print("Compression factor:", metrics['rho'])
        print("Compression RMSE:", metrics['rmse'])

# Compressing an image via principal component analysis.
# Calculated using SVD.
def pca_compression(img, n_comp):
    k = n_comp

    # Divide in 3 channels (BGR) and apply SVD to each one.
    USVb = np.linalg.svd(img[:,:,0], full_matrices=False)
    USVg = np.linalg.svd(img[:,:,1], full_matrices=False)
    USVr = np.linalg.svd(img[:,:,2], full_matrices=False)

    U = np.dstack((USVb[0],USVg[0],USVr[0]))
    S = np.dstack((USVb[1],USVg[1],USVr[1]))
    V = np.dstack((USVb[2],USVg[2],USVr[2]))

    # Consider k components and combine channels.
    out = np.zeros(img.shape)

    Ug = U[:,:k,:]
    Sg = S[:k,:k,:]
    Vg = V[:k,:,:]

    for i in range(img.shape[2]):
        out[:,:,i] = (Ug[:,:,i]*Sg[:,:,i]) @ Vg[:,:,i]

    return out

# Evaluates compression by RMSE and compression factor.
def eval_compression(f, g, f_path, g_path, factor=True):
    rho = getsize(g_path)/getsize(f_path) if factor else 0

    diff = (f-g)**2
    rmse = np.sqrt(diff.sum().sum()/(f.shape[0] * f.shape[1]))

    return {'rho':rho, 'rmse':rmse}

# Plots component graph to verify optimal amount of components to compress.
def plot_components(img, amount):
    rmse = np.zeros(amount)
    for i in range(1,amount+1):
        out = pca_compression(img, i)
        met = eval_compression(img, out, '', '', factor=False)
        rmse[i-1] = met['rmse']

    plt.plot(range(1,amount+1), rmse, 'bo')
    plt.xlabel('Component Amount')
    plt.ylabel('RMSE')
    plt.title('Compression error per component amount')

# Self-explanatory
def show_image(img, name='Image'):
    imshow(name, img)
    waitKey(0)
    destroyAllWindows()

def save_image(img, name, folder):
    imwrite(join(folder, name), img)

if __name__ == "__main__":
    main()


#! /usr/bin/env python3
#
#
# ----------------------------------
# 2018-05-03T13:05:25.563Z
# Luc Blassel
#
# Prepares data by aligning aa sequences together and encoding them before splitting them into training and testing data
# ----------------------------------
#

import pandas as pd
import numpy as np
import os
import pickle
# import pdb

from ast import literal_eval
from category_encoders import OneHotEncoder
from utils.myUtils import createLogger
from sklearn.utils import shuffle

from Bio.Alphabet import IUPAC

# temporary debugging imports
# from time import sleep
# from pprint import pprint

SEQUENCE_ID_COL = 'inputSequence'
ALIGNED_ID_COL = 'alignedSequence'

REF_AA_COL = 'refAALine'
MUTATION_SEQ_COL = 'mutationLine'
FIRST_AA_COL = 'firstAA'
LAST_AA_COL = 'lastAA'
SUBTYPE_COL = "subtypeText"

SAVED_DATASET = 'merged-datasets'
ALIGNED_SET = 'aligned-raw-dataset'
NAIVE_LABEL = 'naive'
ENCODED_LABEL = 'encoded_label'

ENCODERS = ['integer', 'bitwise', 'index', 'properties', 'None', 'OneHot']

LOG_LEVEL = 'info'
logger = None

COUNTER = 0

def save(df, outDir, filename, sep):
    df.to_csv(open(os.path.join(outDir, filename), 'w'), sep=sep)

def reader(filepaths):
    """
    reads data from tab delimited files (from sdrmHunter with alignedAAs mod) of naive and treated samples generates dataframe as dataset for ML
    inputs: filepath dict (keys = naive/treated)
    returns: dataframe containing sequence ID, SDRMs, aligned AA sequence and class of sample
    """
    master_df = None

    for path_key in filepaths:

        df = pd.read_csv(filepaths[path_key], sep='\t', header=0)
        df['label'] = path_key
        df = df.set_index(SEQUENCE_ID_COL, drop=False)

        if master_df is None:
            master_df = df
        else:
            master_df = pd.concat([master_df, df])

        logger.info("{0} class, {1} samples. ".format(path_key, df.shape[0]))

    return master_df

def removeMuts(df):
    """
    removes lines that have SDRMs
    """
    sdrms = df.filter(regex='[A-Z]+:[A-Z][0-9]+[A-Z]+', axis=1)

    count = sdrms.any(axis=1)

    return df[~df[SEQUENCE_ID_COL].isin(count[count].index)]

def removePos(df):
    """
    returns positions where at least one sequence in the dataset has a SDRM
    """
    RE = '[A-Z]+:[A-Z][0-9]+[A-Z]+'
    sdrms = [pos.split(':')[1] for pos in df.filter(regex=RE, axis=1).columns.tolist()]

    positions = [''.join([el for el in pos if el.isdigit()]) for pos in sdrms]
    return list(set(positions))

def removePosAlt(aligned_aa):
    """
    returns positions where at least one sequence in the dataset has a mutation from the compiled list
    """
    # import pdb; pdb.set_trace()
    drmsFile = pd.read_csv(os.path.abspath(os.path.join("scripts", "utils", "RT-DRMS.csv"))).set_index("Pos")

    drms = {k: [v for v in drmsFile["Mut"].loc[k] if v not in ['d', 'i']] for k in drmsFile.index}
    drms = {k: v for (k, v) in drms.items() if v}
    posDf = pd.DataFrame(index=aligned_aa.index, columns=["empty"])
    for pos in drms.keys():
        posDf[str(pos)] = aligned_aa[ALIGNED_ID_COL].apply(lambda x: pos < len(x) and x[pos-1] in drms[pos])

    posToRemove = posDf.drop("empty", axis=1).any(axis=0).index.tolist()
    return posToRemove

def removePosOneHot(aligned_aa):
    """
    removes specific pos+AA from drm list
    """
    drmsFile = pd.read_csv(os.path.abspath(os.path.join("scripts", "utils", "RT-DRMS.csv"))).set_index("Pos")
    wt = list(set(["{}_{}".format(pos, mut) for pos in drmsFile.index for mut in drmsFile["WT"].loc[pos]]))
    drms = ["{}_{}".format(pos, mut) for pos in list(set(drmsFile.index.tolist())) for mut in drmsFile["Mut"].loc[pos]]
    return aligned_aa.drop(columns=drms+wt, errors='ignore')

def breakdownSubtypes(df):
    """
    Shows frequencies of subtypes in dataset
    """
    cleaned = df[SUBTYPE_COL].apply(lambda x: x.split("(")[0].strip())
    logger.info("subtypes in the dataset (frequencies):\n{}".format(cleaned.value_counts(normalize=True)))

def chooseSubtype(df, subtype):
    """
    allows the user to specify one subtype, or None
    """
    if subtype.upper() not in ('NONE', 'ALL'):
        cleaned = df[SUBTYPE_COL].apply(lambda x: x.split("(")[0].strip())
        return df[cleaned == subtype]
    else:
        return df

def processDataset(df, encoding_method, subtype, outDir, remove):
    """
    readies the data for training of Random Forest Classifier, aligning sequences, encoding non numerical data.
    inputs: Dataframe containing all the dataset
    returns: training set  and testing set features and labels as np arrays
    """

    def alignEnds(seq, firstAA, lastAA, start, end):
        """
        aligns sequences that are shorter at the ends than the reference sequence
        """

        if firstAA > start:
            seq = [" 0 " for i in range(firstAA - start)] + seq
        if lastAA < end:
            seq += [" 0 " for i in range(end - lastAA)]
        return seq

    if os.path.exists(os.path.join(outDir, SAVED_DATASET+'-'+encoding_method+'-'+subtype+'.tab')):  # avoid costly processing
        df = pd.read_csv(os.path.join(outDir, SAVED_DATASET+'-'+encoding_method+'-'+subtype+'.tab'), sep='\t')
        aligned_aa = df[ALIGNED_ID_COL].apply(literal_eval)
        logger.info('loaded saved dataset from disk.')
        position = [str(i) for i in list(range(1, len(aligned_aa[0])+1))]
        # posToRemove = removePos(df)
        # posToRemove = removePosAlt(aligned_aa)
        # posToRemoveInt = [int(pos) - 1 for pos in posToRemove]

    else:

        start = int(min(df[FIRST_AA_COL]))
        end = int(max(df[LAST_AA_COL]))

        cols = [REF_AA_COL, MUTATION_SEQ_COL, FIRST_AA_COL, LAST_AA_COL]
        df = df.dropna(subset=cols)
        breakdownSubtypes(df)
        df = chooseSubtype(df, subtype)

        df[REF_AA_COL] = df[REF_AA_COL].apply(literal_eval)
        df[MUTATION_SEQ_COL] = df[MUTATION_SEQ_COL].apply(literal_eval)

        df[FIRST_AA_COL] = df[FIRST_AA_COL].apply(int)
        df[LAST_AA_COL] = df[LAST_AA_COL].apply(int)

        # posToRemove = removePos(df)
        # posToRemoveInt = [int(pos) - 1 for pos in posToRemove]

        aligned_aa = df[cols]

        df = df.drop(labels=cols, axis=1)
        aligned_aa[MUTATION_SEQ_COL] = aligned_aa.apply(lambda row: alignEnds(row[MUTATION_SEQ_COL], row[FIRST_AA_COL], row[LAST_AA_COL], start, end), axis=1)

        ref = aligned_aa[REF_AA_COL].loc[aligned_aa[REF_AA_COL].apply(len).idxmax()]
        seqs = np.array([seq for seq in aligned_aa[MUTATION_SEQ_COL]])

        aligned, position = align(ref, seqs)
        aligned_aa[ALIGNED_ID_COL] = aligned.tolist()
        aligned_aa = aligned_aa.drop([MUTATION_SEQ_COL, REF_AA_COL, FIRST_AA_COL, LAST_AA_COL], axis=1)

        logger.debug('created aligned dataset, ready to be encoded.')

        # df.join(aligned_aa).to_csv(os.path.join(outDir,SAVED_DATASET+'_'+encoding_method+'.tab', sep='\t'))
        save(df.join(aligned_aa), outDir, SAVED_DATASET+'-'+encoding_method+'-'+subtype+'-'+str(remove)+'.tab', '\t')
        logger.debug('saved dataset to disk.')

    df[ENCODED_LABEL] = [0 if label == NAIVE_LABEL else 1 for label in df['label']]

    # posToRemove = removePosAlt(aligned_aa)
    # posToRemoveInt = [int(pos) - 1 for pos in posToRemove]

    features = aligned_aa.values

    # HACK find a better way to do this...
    features = np.array([np.array([np.array(arr) for arr in seq]).flatten() for seq in features])

    # if remove:
    #     #removing positions presenting SDRMs from dataset
    #     features = np.delete(features, posToRemoveInt, axis=1)
    #     position = np.delete(position, posToRemoveInt)

    if not os.path.exists(ALIGNED_SET+'_'+subtype+'_'+str(remove)+'.tab'):
        toSave = pd.DataFrame(features)
        toSave.columns = position
        toSave[SEQUENCE_ID_COL] = df[SEQUENCE_ID_COL].values
        toSave = toSave.set_index(SEQUENCE_ID_COL, drop=True)
        toSave = df.set_index(SEQUENCE_ID_COL)[[ENCODED_LABEL]].join(toSave)
        save(toSave, outDir, ALIGNED_SET+'_'+subtype+'_'+str(remove)+'.tab', '\t')

    if encoding_method != 'None' and encoding_method != 'OneHot':
        features = encode(features, encoding_method, outDir)
    elif encoding_method == 'OneHot':
        features = pd.DataFrame(features)
        features.columns = position
        enc = OneHotEncoder(use_cat_names=True, handle_unknown='ignore')
        features = enc.fit_transform(features)

    features = pd.DataFrame(features)
    if encoding_method not in ('bitwise', 'properties', 'OneHot'):
        features.columns = position
    df = df.set_index(SEQUENCE_ID_COL)
    features[SEQUENCE_ID_COL] = df.index

    features = features.set_index(SEQUENCE_ID_COL, drop=True)

    if remove:
        features = removePosOneHot(features)
    # df = df.set_index(SEQUENCE_ID_COL, drop=True)

    fullSet = df[[ENCODED_LABEL, 'label']].join(features)

    # setNaive = fullSet[fullSet[ENCODED_LABEL] == 0]
    # setTreated = fullSet[fullSet[ENCODED_LABEL] == 1]
    #
    # setNaiveDownsampled = resample(setNaive, replace=False, n_samples=setTreated.shape[0])
    #
    # fullSet = pd.concat([setTreated, setNaiveDownsampled])

    # fullSet['isTrain'] = np.random.uniform(0,1, len(fullSet)) <= .95
    fullSet = shuffle(fullSet)

    # train = fullSet[fullSet['isTrain'] == True]
    # test =  fullSet[fullSet['isTrain'] != True]
    #
    #
    # logger.debug('train and test sets:')
    # logger.debug('train: \n{}'.format(train))
    # logger.debug('test:  \n{}'.format(test ))

    # return train, test
    return fullSet

def encode(features, encoding_method, outDir):
    """
    encodes a set of AA sequences as a sequence of 5 bits
    Inputs: array containing sequences, shape (n, d)
    outputs: array containing encoded sequences, shape (n, d*5)
    """

    def integerEncoder():
        """
        creates encoding dictionnary for amino acids, each AA is given a number which is encoded as an Integer.
        inputs: None
        outputs: dict with key -> AA letter, value -> encoded number
        """

        letters = IUPAC.protein.letters
        letters = '-'+letters+'X*'  # adding gap, mixed AA and stop codon to alphabet
        logger.debug('using simple numerical encoder')
        encoder = {letter: letters.index(letter) for letter in letters}

        return encoder

    def bitwiseEncoder():
        """
        creates encoding dictionnary for amino acids, each AA is given a number which is encoded in 5 bits.
        inputs: None
        outputs: dict with key -> AA letter, value -> encoded number
        """
        letters = IUPAC.protein.letters
        letters = '-'+letters+'X*'  # adding gap, mixed AA and stop codon to alphabet
        logger.debug('using 5bit numerical encoder')
        encoder = {letter: '{0:05b}'.format(letters.index(letter)) for letter in letters}

        return encoder

    def indexEncoder():
        """
        creates encoding dictionnary based on physico-chemical properties of AAs contained in aaindex1 file
        inputs: None
        outputs: dict w/ key -> AA Letter, value -> encoded value
        """

        import utils.AAindex as aai

        records = {}
        logger.debug('using Physico-chemical properties encoder')

        # properties in AAindex id format
        record_id = ['JURD980101',
                     'HOPT810101',
                     'MCMT640101',
                     'BHAR880101',
                     'FAUJ880103',
                     'BULH740101',
                     'COSI940101',
                     'RADA880101',
                     'ZIMJ680104'
                     ]

        for id in record_id:
            dic, rec = aai.GetAAIndex1(name=id, path=outDir)
            records[id] = dic

        # adding all property values
        encoder = {}
        for key in records[record_id[0]]:
            for dic in records.values():
                if key not in encoder.keys():
                    encoder[key] = dic[key]
                else:
                    encoder[key] += dic[key]

        mean = 0
        for key in encoder:
            mean += encoder[key]

        mean /= len(encoder.keys())

        # arbitrary #TODO find a better way of dealing with this?
        encoder['X'] = mean
        encoder['-'] = 0
        encoder['*'] = 0

        return encoder

    def propertyEncoder():
        """
        creates encoder based on appartenance to 14 physico-chemical propperty defined groups (ie. aliphatic, aromatic, charged, tiny, etc...)
        """
        # necesseray to keep order of binary properties consistnat accross runs
        order = ['tiny', 'small', 'aliphatic', 'charged', 'negative', 'positive', 'polar', 'hydrophobic', 'extra-tiny', 'pentagonal', 'hexagonal', 'forked', 'crossed']
        groups = {'tiny': 'AGS',
                  'small': 'ANDCGPSTV',
                  'aliphatic': 'HFWY',
                  'charged': 'RDQHK',
                  'negative': 'DQ',
                  'positive': 'RHK',
                  'polar': 'RNDQEHKSTWY',
                  'hydrophobic': 'ACGHILKMFTWYV',
                  'extra-tiny': 'G',
                  'pentagonal': 'HPW',
                  'hexagonal': 'FWY',
                  'forked': 'RNDQELV',
                  'crossed': 'IST'}

        letters = IUPAC.protein.letters
        encoder = {k: [] for k in letters}
        for property in order:
            members = groups[property]
            for letter in letters:
                encoder[letter] += [1] if letter in members else [0]
        encoder['X'] = [1 for i in range(len(groups))]
        encoder['*'] = [0 for i in range(len(groups))]
        encoder['-'] = encoder['*']

        return encoder

    # set encoder
    if encoding_method == 'integer':
        encoder = integerEncoder()
    elif encoding_method == 'bitwise':
        encoder = bitwiseEncoder()
    elif encoding_method == 'index':
        encoder = indexEncoder()
    else:
        encoder = propertyEncoder()

    i = 0
    encoded = [[] for i in features]

    for i in range(len(features)):
        for aa in features[i]:
            if encoding_method in ('bitwise', 'properties'):
                encoded[i] += list(encoder[aa])
            else:
                encoded[i] += [encoder[aa]]
    return np.array([np.array(seq) for seq in encoded])

def align(ref, seqs):
    """
    formats aligned sequences into an array (according to reference consensus sequence)
    inputs: ref -> reference sequence array of AA characters,
            seqs -> aligned sequences as 2D array of strings (from HIVdB Sierra)
    outputs: array of aligned sequences (array of chars)
    """

    def calcFreq(df):
        """
        calculates frequencies of mutations for a given set of sequences at all positions
        """
        freqs = []
        for i in range(df.shape[1]):
            unique, counts = np.unique(df[:, i], return_counts=True)
            posFreq = {k: v for k, v in zip(unique, counts)}
            for key, val in zip(unique, counts):
                if len(key) > 1:
                    for letter in key:
                        if letter in posFreq.keys():
                            posFreq[letter] += val
                        else:
                            posFreq[letter] = val
            posFreq = {k: v for k, v in posFreq.items() if len(k) == 1}
            freqs += [posFreq]
        return freqs

    def findMax(dic, val):  # TODO consider apparition of AA in other mixed mutations
        """
        Finds most likly mutated AA for mixed mutation sites
        """
        mostLikely = ''
        maxCount = 0

        for AA in val:
            if AA in dic.keys():
                if dic[AA] > maxCount:
                    mostLikely = AA
                    maxCount = dic[AA]
        if mostLikely == '':
            logger.debug('weird behaviour at this position\n{}'.format(dic))
            return 'X'
        return mostLikely

    alignedSeqs = [[] for i in range(len(seqs))]

    # to keep AA position in reference sequence
    position = set()
    mainPos = 0

    # remove whitespace from sierra webservice output
    cleanup = np.vectorize(lambda val: val.strip())
    encodeDels = np.vectorize(lambda val: '0' if val == 'Del' else val)

    seqs = cleanup(seqs)
    seqs = encodeDels(seqs)
    ref = cleanup(ref)

    # lengths of longest addition mutation at each position of consensus AA sequence
    lengths = np.apply_along_axis(np.vectorize(lambda x: len(x)), 1, seqs)
    maxes = [max(lengths[:, i]) for i in range(len(lengths[0]))]

    freqs = calcFreq(seqs)

    for i in range(len(ref)):
        mainPos += 1
        position.add(str(mainPos))
        for j in range(len(seqs)):
            if seqs[j][i] == '-':  # AA from consensus sequence
                alignedSeqs[j] += ref[i]
            elif seqs[j][i] == '0':
                alignedSeqs[j] += '-'
            elif len(seqs[j][i]) > 1:  # mixed AA
                aa = findMax(freqs[i], seqs[j][i])
                alignedSeqs[j] += aa
                logger.debug('Most likely aa: {}'.format(aa))
            else:  # substituted AA
                alignedSeqs[j] += seqs[j][i]

    logger.info('aligned sequences to consensus sequence and each other.')
    return np.array(alignedSeqs), sorted(list(position), key=float)

def main():
    global logger
    import argparse

    def valid_method(encoding_method):
        if encoding_method in ENCODERS:
            return encoding_method
        else:
            raise argparse.ArgumentTypeError('{} is not a valid encoding method. methods must be one of these: \n {}'.format(encoding_method, ENCODERS))

    def str2bool(v):
        if v.lower() in ('yes', 'true', 't', 'y', '1'):
            return True
        elif v.lower() in ('no', 'false', 'f', 'n', '0'):
            return False
        else:
            raise argparse.ArgumentTypeError('Boolean value expected.')

    parser = argparse.ArgumentParser(description='processes data extracted from HIVdB for machine learning')
    parser.add_argument('--naive',   help='tab delimited file containing SDRMs as well as Aligned AA sequences for naive samples',   required=True)
    parser.add_argument('--treated', help='tab delimited file containing SDRMs as well as Aligned AA sequences for treated samples', required=True)
    parser.add_argument('--encoding', help='method of encoding you want within these options: \n{}'.format(ENCODERS), required=True, type=valid_method)
    parser.add_argument('--outdir', help='directory where you want output files to be created', required=False)
    parser.add_argument('--subtype', help='If you want to restrict the data to a certain subtype', required=False, nargs='?', const="All", type=str)
    parser.add_argument('--remove', help='If you want to remove positions where SDRMs occur', required=True, type=str2bool)
    parser.add_argument('--log', help='filepath to log file', default='processing.log', required=False)
    params = parser.parse_args()

    logger = createLogger(params.log, LOG_LEVEL)

    logger.info('\n\n' + 100 * '=' + '\n\n')
    logger.debug("parameters: \n{}".format(params))

    filepaths = {'naive': params.naive, 'treated': params.treated}
    logger.info('Processing from these files:\n{}'.format(filepaths))
    df = reader(filepaths)

    outDir = ''
    if params.outdir is not None:
        outDir = params.outdir

    # train_set, test_set = processDataset(df, params.encoding, outDir)
    train_set = processDataset(df, params.encoding, params.subtype, outDir, params.remove)

    pickle.dump(train_set, open(os.path.join(outDir, 'train-'+params.encoding+'-'+params.subtype+'-'+str(params.remove)+'.pkl'), 'wb'))
    # pickle.dump(test_set,  open(os.path.join(outDir,'test_'+params.encoding+'.pkl'), 'wb' ))

    logger.info('proportion of treated sequences in datasets:')
    logger.info("train: {}\n num_treated :{} num_total: {}".format(np.sum(train_set[ENCODED_LABEL])/len(train_set), train_set[train_set[ENCODED_LABEL] == 1].shape[0], train_set.shape[0]))
    # logger.info("test: {}\n num_treated :{} num_total: {}".format(np.sum(test_set[ENCODED_LABEL])/len(test_set), test_set[test_set[ENCODED_LABEL]==1].shape[0], test_set.shape[0]))
    logger.info("total: {}".format(np.sum([0 if label == NAIVE_LABEL else 1 for label in df['label']])/df.shape[0]))

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        logger.exception(e)


#!/usr/bin/python3
# -*- coding: utf-8 -*-
import sys
from PyQt5.QtWidgets import (QWidget, QLabel, QLineEdit,
                             QTextEdit, QGridLayout, QApplication, QPushButton, QFileDialog, QMessageBox)
import calendar
import xlsxwriter
import datetime

from qtpy import QtGui

class Example(QWidget):

    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        self.rir = []
        now = datetime.datetime.now()
        holidays = {datetime.date(now.year, 8, 14)}  # you can add more here
        businessdays = 0
        for i in range(1, 32):
            try:
                thisdate = datetime.date(now.year, now.month, i)
            except(ValueError):
                break
            if thisdate.weekday() < 5 and thisdate not in holidays:  # Monday == 0, Sunday == 6
                businessdays += 1
                self.rir.append(thisdate.day)
        self.mth = now.strftime("%m")
        self.yr = now.strftime("%Y")
        self.title = QLabel('Всего дней в месяце: '+ str(calendar.monthrange(int(self.yr), int(self.mth))[1]) )
        self.author = QLabel('Всего рабочих дней: '+ str(businessdays))
        self.review = QLabel('Числа тех дней по которым работаем: ' + str(self.rir))
        self.btn = QPushButton("Создание отчёта", self)
        self.btn.clicked.connect(self.buttonClicked)
        grid = QGridLayout()
        grid.setSpacing(10)
        grid.addWidget(self.title, 1, 0)
        grid.addWidget(self.author, 2, 0)
        grid.addWidget(self.review, 3, 0)
        grid.addWidget(self.btn, 4 , 0 )
        self.setLayout(grid)
        self.setGeometry(300, 300, 150, 100)
        self.setWindowTitle('Генератор отчётов')
        self.show()

    def  buttonClicked(self):
        try:
          options = QFileDialog.Options()
          options |= QFileDialog.DontUseNativeDialog
          fileName, _ = QFileDialog.getSaveFileName(self,"QFileDialog.getSaveFileName()","","MS Excel Files (*.xlsx);;All Files (*)", options=options)
          workbook = xlsxwriter.Workbook(fileName + '.xlsx')
          worksheet = workbook.add_worksheet()
          worksheet.set_column('A:A', 30)
          worksheet.set_column('B:B',120)
          bold = workbook.add_format({'bold': True})
          bold.set_font_color('red')
          cell_format = workbook.add_format({'bold': True, 'bg_color': 'black'})
          cell_format2 = workbook.add_format({'bold': True, 'bg_color': '#e1f6fa '})
          i = 0
          i2 = 13
          worksheet.write('A' + str(i + 1),str(self.rir[0])+ "." + self.mth + "." + self.yr,bold)
          worksheet.write('B' + str(i + 1),'',cell_format)
          i = i+1
          ti = 0
          toto = 0
          day = 0
          merge_format = workbook.add_format({
            'bold': True,
            'border': 4,
            'align': 'left',
            'valign': 'vcenter',
            'fg_color': '#e1f6fa',
          })
          time_massive = ['10-11:' ,'11-12:' ,'12-12:30:' ,'12:30-13:30:' ,'13:30-14:30:' ,'14:30-15:30:' ,'15:30-16:00:','16-17:' ,'17-18:', 'Протоколы:', 'Изучение библиотеки:', 'Разработка программы:']

          while  i+1 <= len(self.rir):

            worksheet.write('A' + str(i + i2 + 1), str(self.rir[i]) + "." + self.mth + "." + self.yr,bold)
            worksheet.write('B' + str(i + i2 + 1),'', cell_format)

            ti = 0
            while ti + 1 <= len(time_massive):

                 worksheet.write('A' + str(i + i2 + 1 + ti + 1), time_massive[ti])
                 worksheet.write('A' + str(ti+2), time_massive[ti])
                 worksheet.write('B' + str(i + i2 + 1 + ti + 1), '',merge_format)
                 worksheet.write('B' + str(ti+2), '',merge_format)
                 worksheet.write('B' + str(i + i2 - 10), "Обед", merge_format)
                 worksheet.write('B' + str(i + i2 - 6), "Перерыв", merge_format)
                 ti = ti + 1
                 toto = toto + 1

            toto = toto + 1
            i = i +1
            i2 = i2 + 13
          i = 0
          print("1 " + str(i + i2 + 1 + ti + 1))
          print("2 " + str(toto))
          print("3 " + str(i2))
          worksheet.write("B"+ str(4),'Обед',merge_format)
          worksheet.write("B"+ str(8),'Перерыв',merge_format)
          worksheet.write("B"+ str(4),'Обед',merge_format)
          worksheet.write("B"+ str(8),'Перерыв',merge_format)
          worksheet.write("B"+ str(i + i2 + 1 + ti - 1),'Обед',merge_format)
          worksheet.write("B"+ str(i + i2 + 1 + ti + 3),'Перерыв',merge_format)
          #worksheet.write('B' + str(i + i2 - 6), "Перерыв", merge_format)
          workbook.close()
          Qmessa("Готово")
        except:
          Qmessa("Произошла ошибка")

class Qmessa(QMessageBox):
    def __init__(self, a):
        super().__init__()
        msg = QMessageBox()
        msg.setWindowTitle("Info")
        msg.setText(a)
        result = msg.setStandardButtons(QMessageBox.Ok)
        retval = msg.exec_()

if __name__ == '__main__':

    app = QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())


from socket import *
import os,sys

class FTPClient():
    def __init__(self,sockfd):
        self.sockfd = sockfd
    def do_list(self):
        self.sockfd.send(b'L')
        #等待回复:
        data = self.sockfd.recv(128).decode()
        if data == 'OK':
            while True:
                file = self.sockfd.recv(1024).decode()
                if file == '##':
                    break
                file = file.strip().split(' ')
                for line in file:
                    print(line)
        else:
            #无法操作
            print(data)

    def do_quit(self):
        self.sockfd.send(b'Q')
        self.sockfd.close()
        sys.exit("退出客户端")
def main():
    if len(sys.argv) < 3:
        sys.exit("格式有误")
    HOST = sys.argv[1]
    PORT = int(sys.argv[2])
    ADDR = (HOST,PORT)

    s = socket()
    try:
        s.connect(ADDR)
    except Exception as e:
        print("服务器异常",e)
        return

    #创建封装套接字类的对象
    ftp = FTPClient(s)
    while True:
        #展示页面
        print("========张奔版GUI下载工具 V8.92========")
        print("****    list        ****")
        print("****    get file    ****")
        print("****    put file    ****")
        print("****    quit        ****")
        print("=====================================")

        cmd = input("请输入命令:")
        if cmd.strip() =='list':
            ftp.do_list()
        elif cmd.strip() == 'quit':
            ftp.do_quit()

main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@package cgsn_parsers.parsers.parse_phsen
@file cgsn_parsers/parsers/parse_phsen.py
@author Christopher Wingard
@brief Parses PHSEN data logged by the custom built WHOI data loggers.
'''
import os
import re

# Import common utilites and base classes
from cgsn_parsers.parsers.common import ParserCommon
from cgsn_parsers.parsers.common import dcl_to_epoch, inputs, DCL_TIMESTAMP

# Regex pattern for a line with a DCL time stamp, the "*" character, 4 unknown
# characters (2 for a 1 byte hash of the unit serial number and calibration,
# and 2 for the length byte), and a '0A' (indicating a Type 10, or pH, data
# record), all of which combine to denote the start of a sampling record.
PATTERN_START = (
    DCL_TIMESTAMP + r'\s+' +                # DCL Time-Stamp
    r'(\*[A-F0-9]{4}0A[A-F0-9]+)' + r'\s'   # Beginning of a pH sampling record
)
REGEX_START = re.compile(PATTERN_START, re.DOTALL)

_parameter_names_phsen = [
        'dcl_date_time_string',
        'record_length',
        'record_type',
        'record_time',
        'thermistor_start',
        'reference_measurements',
        'light_measurements',
        'voltage_battery',
        'thermistor_end'
    ]

class Parser(ParserCommon):
    """
    A Parser subclass that calls the Parser base class, adds the PHSEN specific
    methods to parse the data, and extracts the PHSEN data records from the DCL
    daily log files.
    """
    def __init__(self, infile):
        self.initialize(infile, _parameter_names_phsen)

    def parse_data(self):
        '''
        Iterate through the record markers (defined via the regex expression
        above) in the data object, and parse the data file into a pre-defined
        dictionary object created using the Bunch class.
        '''
        record_marker = [m.start() for m in REGEX_START.finditer(self.raw)]

        # if we have found record markers, work through the data
        while record_marker:
            # for each record marker, set the start and stop points of the sample
            start = record_marker[0]
            if len(record_marker) > 1:
                # stopping point is the next record marker
                stop = record_marker[1]
            else:
                # stopping point is the end of the file
                stop = len(self.raw)

            # now create the sample string
            sample = self.raw[start:stop]

            # pull out of the sample string the DCL timestamp
            timestamp = REGEX_START.match(sample).group(1)

            # clean up the rest of the extraneous strings in the sample string
            sample = re.sub(DCL_TIMESTAMP, '', sample)  # get rid of the extra time stamps
            sample = re.sub(r'\s+', '', sample)         # get rid of newlines and spaces
            sample = re.sub(r'\[\w+:\w+\]:.+', '', sample)  # get rid of the stop messages

            # if we have a complete sample, process it.
            if len(sample) == 465:
                # print '%s --- %s\n' % (timestamp, sample)
                self._build_parsed_values(timestamp, sample)

            # bump to the next marker
            record_marker.pop(0)

    def _build_parsed_values(self, timestamp, sample):
        """
        Extract the data from the relevant regex groups and assign to elements
        of the data dictionary.
        """
        # Use the date_time_string to calculate an epoch timestamp (seconds since
        # 1970-01-01)
        epts = dcl_to_epoch(timestamp)
        self.data.time.append(epts)
        self.data.dcl_date_time_string.append(timestamp)

        self.data.record_length.append(int(sample[3:5], 16))
        self.data.record_type.append(int(sample[5:7], 16))
        self.data.record_time.append(int(sample[7:15], 16))
        self.data.thermistor_start.append(int(sample[15:19], 16))

        cnt = 19
        reference = []  # create empty list to hold the 16 reference measurements
        for i in range(0, 16):
            indx = (i * 4) + cnt
            reference.append(int(sample[indx:indx+4], 16))

        self.data.reference_measurements.append(reference)

        cnt = indx + 4  # reset the counter to start with the light measurements
        light = []  # create empty list to hold the 92 light measurements
        for i in range(0, 92):
            indx = (i * 4) + cnt
            light.append(int(sample[indx:indx+4], 16))

        self.data.light_measurements.append(light)

        cnt = indx + 8  # reset the counter for the final parameters
        self.data.voltage_battery.append(int(sample[cnt:cnt+4], 16))
        self.data.thermistor_end.append(int(sample[cnt+4:cnt+8], 16))

if __name__ == '__main__':
    # load the input arguments
    args = inputs()
    infile = os.path.abspath(args.infile)
    outfile = os.path.abspath(args.outfile)

    # initialize the Parser object for PHSEN
    phsen = Parser(infile)

    # load the data into a buffered object and parse the data into a dictionary
    phsen.load_binary()     # not really binary, but this creates on object that is easy to parse
    phsen.parse_data()

    # write the resulting Bunch object via the toJSON method to a JSON
    # formatted data file (note, no pretty-printing keeping things compact)
    with open(outfile, 'w') as f:
        f.write(phsen.data.toJSON())


# Import libraries
import cv2
import numpy as np
from dataPath import DATA_PATH

# Read image
image = cv2.imread(DATA_PATH+"images/boy.jpg")

brightnessOffset = 50

# Add the offset for increasing brightness
brightHighOpenCV = cv2.add(image, np.ones(image.shape,dtype='uint8')*brightnessOffset)

brightHighInt32 = np.int32(image) + brightnessOffset
brightHighInt32Clipped = np.clip(brightHighInt32,0,255)

# Display the outputs
#cv2.imshow("original image",image)
#cv2.imshow("using cv2.add function",brightHighOpenCV)
#cv2.imshow("Using numpy and clipping", brightHighInt32Clipped)
#cv2.waitKey(0)
cv2.imwrite("../results/brightHighOpenCV.png",brightHighOpenCV)
cv2.imwrite("../results/brightHighInt32Clipped.png",brightHighInt32Clipped)

# Add the offset for increasing brightness
brightHighFloat32 = np.float32(image) + brightnessOffset
brightHighFloat32NormalizedClipped = np.clip(brightHighFloat32/255,0,1)

brightHighFloat32ClippedUint8 = np.uint8(brightHighFloat32NormalizedClipped*255)

# Display the outputs
#cv2.imshow("original image",image)
#cv2.imshow("Using np.float32 and clipping",brightHighFloat32NormalizedClipped)
#cv2.imshow("Using int->float->int and clipping", brightHighFloat32ClippedUint8)
#cv2.waitKey(0)
cv2.imwrite("../results/brightHighFloat32NormalizedClipped.png",brightHighFloat32NormalizedClipped)
cv2.imwrite("../results/brightHighFloat32ClippedUint8.png",brightHighFloat32ClippedUint8)


import argparse
import os
import os.path as osp
import shutil
import tempfile
import pdb
import numpy as np
import pickle

import mmcv
import torch
import torch.distributed as dist
from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
from mmcv.runner import get_dist_info, load_checkpoint

from mmdet.apis import init_dist
from mmdet.core import tao_eval, results2json, wrap_fp16_model
from mmdet.datasets import build_dataloader, build_dataset, FilesDataset
from mmdet.models import build_detector

from mmdet.core import build_assigner

import pickle
import common
from inference_utils import *

def single_gpu_test(model, data_loader, show=False, cfg=None):
    model.eval()
    results = []
    dataset = data_loader.dataset
    prog_bar = mmcv.ProgressBar(len(dataset))
    for i, data in enumerate(data_loader):
        with torch.no_grad():
            result = model(return_loss=False, rescale=not show, **data)
        results.append(result)

        if show:
            model.module.show_result(data, result)

        batch_size = data['img'][0].size(0)
        for _ in range(batch_size):
            prog_bar.update()
    return results

def multi_gpu_test(model, data_loader, tmpdir=None):
    model.eval()
    results = []
    dataset = data_loader.dataset
    rank, world_size = get_dist_info()
    if rank == 0:
        prog_bar = mmcv.ProgressBar(len(dataset))
    for i, data in enumerate(data_loader):
        with torch.no_grad():
            result = model(return_loss=False, rescale=True, **data)
        results.append(result)

        if rank == 0:
            batch_size = data['img'][0].size(0)
            for _ in range(batch_size * world_size):
                prog_bar.update()

    # collect results from all ranks
    results = collect_results(results, len(dataset), tmpdir)

    return results

def collect_results(result_part, size, tmpdir=None):
    rank, world_size = get_dist_info()
    # create a tmp dir if it is not specified
    if tmpdir is None:
        MAX_LEN = 512
        # 32 is whitespace
        dir_tensor = torch.full((MAX_LEN, ),
                                32,
                                dtype=torch.uint8,
                                device='cuda')
        if rank == 0:
            tmpdir = tempfile.mkdtemp()
            tmpdir = torch.tensor(
                bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda')
            dir_tensor[:len(tmpdir)] = tmpdir
        dist.broadcast(dir_tensor, 0)
        tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip()
    else:
        mmcv.mkdir_or_exist(tmpdir)
    # dump the part result to the dir
    mmcv.dump(result_part, osp.join(tmpdir, 'part_{}.pkl'.format(rank)))
    dist.barrier()
    # collect all parts
    if rank != 0:
        return None
    else:
        # load results of all parts from tmp dir
        part_list = []
        for i in range(world_size):
            part_file = osp.join(tmpdir, 'part_{}.pkl'.format(i))
            part_list.append(mmcv.load(part_file))
        # sort the results
        ordered_results = []
        for res in zip(*part_list):
            ordered_results.extend(list(res))
        # the dataloader may pad some samples
        ordered_results = ordered_results[:size]
        # remove tmp dir
        shutil.rmtree(tmpdir)
        return ordered_results

def parse_args():
    parser = argparse.ArgumentParser(description='MMDet test detector')
    parser.add_argument('--config', type=str, default='/home/songbai.xb/workspace/research/detection/BalancedGroupSoftmax/configs/tao/gs_htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_lvis.py', help='test config file path')
    parser.add_argument('--checkpoint', type=str, default='/home/songbai.xb/workspace/research/detection/BalancedGroupSoftmax/data/pretrained_models/gs_htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_lvis.pth', help='checkpoint file')
    parser.add_argument('--img_dir', type=str, default='', help='inference img dir')
    parser.add_argument('--img_list', type=str, default='', help='inference img dir')
    parser.add_argument('--out_dir', type=str, default='./tmp/file/train_nonms_tta/', help='output dir')
    parser.add_argument("-b", "--batch_size", default=1, type=int, help="batch_size default 32")
    parser.add_argument("-g",'--gpuid', type=str, default='0', help='visible gpu ids')
    parser.add_argument('--tmpdir', help='tmp dir for writing some results')
    parser.add_argument(
        '--launcher',
        choices=['none', 'pytorch', 'slurm', 'mpi'],
        default='none',
        help='job launcher')
    parser.add_argument('--local_rank', type=int, default=0)
    parser.add_argument('--tau', type=float, default=0.0)
    args = parser.parse_args()
    if 'LOCAL_RANK' not in os.environ:
        os.environ['LOCAL_RANK'] = str(args.local_rank)
    return args

def reweight_cls(model, tauuu):

    if tauuu == 0:
        return model

    model_dict = model.state_dict()

    def pnorm(weights, tau):
        normB = torch.norm(weights, 2, 1)
        ws = weights.clone()

        for i in range(0, weights.shape[0]):
            ws[i] = ws[i] / torch.pow(normB[i], tau)

        return ws

    reweight_set = ['bbox_head.fc_cls.weight']
    tau = tauuu
    for k in reweight_set:
        weight = model_dict[k]  # ([1231, 1024])
        weight = pnorm(weight, tau)
        model_dict[k].copy_(weight)
        print('Reweight param {:<30} with tau={}'.format(k, tau))

    return model

def save_in_tao_format(mmdet_res, results_path):
    '''
    Args:
        mmdet_res (list[array]):  一张图的结果
    '''

    boxes_decoded = []
    scores_decoded = []
    classes_decoded = []
    for i, dets in enumerate(mmdet_res):
        for det in dets:
            boxes_decoded.append([det[0], det[1], det[2], det[3]])
            scores_decoded.append(det[4])
            classes_decoded.append(i)

    predictions_decoded = {}
    predictions_decoded["instances"] = {
        "pred_boxes": boxes_decoded,
        "scores": scores_decoded,
        "pred_classes": classes_decoded,
    }

    with open(results_path, 'wb') as f:
        pickle.dump(predictions_decoded, f)

def main():
    args = parse_args()
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpuid

    img_dir = args.img_dir
    out_dir = args.out_dir
    batch_size = args.batch_size

    cfg = mmcv.Config.fromfile(args.config)
    # set cudnn_benchmark
    if cfg.get('cudnn_benchmark', False):
        torch.backends.cudnn.benchmark = True
    cfg.model.pretrained = None
    cfg.data.test.test_mode = True
    # init distributed env first, since logger depends on the dist info.
    if args.launcher == 'none':
        distributed = False
    else:
        distributed = True
        init_dist(args.launcher, **cfg.dist_params)

    # build the dataloader
    if args.img_dir != '':
        file_list = common.load_filepaths(args.img_dir, suffix=('.jpg', '.png', '.jpeg'), recursive=True)
    elif  args.img_list != '':
        file_list = parse_testfile(args.img_list)
    else:
        raise "Both img_dir and img_list is empty."

    dataset = FilesDataset(file_list, cfg.test_pipeline)
    data_loader = build_dataloader(
        dataset,
        imgs_per_gpu=batch_size,
        workers_per_gpu=batch_size,
        dist=distributed,
        shuffle=False)

    # build the model and load checkpoint
    model = build_detector(cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
    fp16_cfg = cfg.get('fp16', None)
    if fp16_cfg is not None:
        wrap_fp16_model(model)
    checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu')

    model = reweight_cls(model, args.tau).cuda()

    model = MMDataParallel(model, device_ids=[0])

    model.eval()
    count = 0
    for i, data in enumerate(data_loader):
        with torch.no_grad():
            # bbox_results, segm_results
            results = model(return_loss=False, rescale=True, **data)

        # batch
        #for result  in results:
        #    file_path = file_list[count]
        #    save_name = file_path.replace('/home/songbai.xb/workspace/projects/TAO/data/TAO/frames/val/', '')
        #    save_path = os.path.join(out_dir, save_name)
        #    common.makedirs(os.path.dirname(save_path))
        #    save_in_tao_format(result, save_path)
        #    count += 1
        file_path = file_list[i]
        save_name = file_path.replace('/home/songbai.xb/workspace/projects/TAO/data/TAO/frames/train/', '')
        save_name = save_name.replace('.jpg', '.pkl').replace('.jpeg', '.pkl')
        save_path = os.path.join(out_dir, save_name)
        common.makedirs(os.path.dirname(save_path))
        save_in_tao_format(results[0], save_path)

if __name__ == '__main__':
    main()


class Solution:
    def isPalindrome(self, s: str) -> bool:
        s = s.lower()
        list1 = []
        for i in range(len(s)):
            if s[i].isalpha() or s[i].isdigit():
                list1.append(s[i])
        a = list(reversed(list1))
        print(a)
        if list1 ==  a:
            return True
        else:
            return False

class Solution:
    def isPalindrome(self, s: str) -> bool:

        list1 = []
        list2 = []
        for i in range(len(s)):
            if ord(s[i]) >= 48 and ord(s[i]) <= 57:
                list1.append(s[i])
            elif s[i] >= 'a' and s[i] <= 'z':
                list1.append(s[i])
            elif s[i] >= 'A' and s[i] <= 'Z':
                list1.append(chr(ord(s[i]) + 32))

        len_list1 = len(list1)

        for i in range(len_list1):
            if list1[i] != list1[len_list1 - 1 - i]:
                return False
        return True


import unittest
from num_occurrences import *

class TestNumOccurrences(unittest.TestCase):
    """Tests for 'mergesort.py'."""

    def setUp(self):
        self.A = []
        self.B = [5]
        self.C = [5, 10, 15]
        self.D = [-2, 2, 2, 2, 4, 7]
        self.E = [1, 1, 1, 1, 4, 5, 7, 7, 9]
        self.F = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8, 9, 10]

    def tearDown(self):
        pass

    def testNumX(self):
        self.assertEqual(0, count_x(self.A, 8))
        self.assertEqual(1, count_x(self.B, 5))
        self.assertEqual(1, count_x(self.C, 5))
        self.assertEqual(3, count_x(self.D, 2))
        self.assertEqual(4, count_x(self.E, 1))
        self.assertEqual(1, count_x(self.E, 5))
        self.assertEqual(2, count_x(self.E, 7))
        self.assertEqual(10, count_x(self.F, 1))

if __name__=='__main__':
    try:
        unittest.main()
    except SystemExit as inst:
        if inst.args[0] is True: # raised by sys.exit(True) when tests failed
            raise


import marcottelab.spectra as spectra
import marcottelab.PSM as PSM
import gzip

def read_inspect_out(filename):
  MS2_list = dict()

  if( filename.endswith('gz') ):
    f_inspect_out = gzip.open(filename,'r')
  else:
    f_inspect_out = open(filename,'r')

  header_idx = dict()
  tmp_idx = 0
  for header in f_inspect_out.readline().strip('#\r\n').split("\t"):
    header_idx[header] = tmp_idx
    tmp_idx += 1

  for line in f_inspect_out:
    tokens = line.strip().split("\t")
    start_scan_id = tokens[ header_idx['Scan#'] ]
    charge = tokens[ header_idx['Charge'] ]
    scan_id = ".".join( [start_scan_id, start_scan_id, charge] )
    peptides = tokens[ header_idx['Annotation'] ].split(".")

    current_hit = PSM.SearchHit()
    current_hit.peptide = "".join(peptides[1:-1])
    current_hit.prev_aa = peptides[0]
    current_hit.next_aa = peptides[-1]
    current_hit.peptide_mass = tokens[ header_idx['PrecursorMZ'] ]
    current_hit.significance = tokens[ header_idx['p-value'] ]
    current_hit.protein_list.append( tokens[ header_idx['Protein'] ] )

    if( not MS2_list.has_key( scan_id ) ):
      MS2_list[scan_id] = spectra.MS2()
      MS2_list[scan_id].charge = charge
      MS2_list[scan_id].start_scan_id = start_scan_id
      MS2_list[scan_id].end_scan_id = start_scan_id
      MS2_list[scan_id].precursor_mass = tokens[ header_idx['PrecursorMZ'] ]

    MS2_list[scan_id].SearchHit_list.append( current_hit )

  f_inspect_out.close()

  return MS2_list


#!/usr/bin/env python3
#coding=utf-8
"""
# Author: huangyisan
# Created Time : 四  5/ 4 00:36:14 2017
# File Name: quick_sort_func.py
# Description:

"""
import time
def quick_sort(array,start,end):
    print("--->",start,end)
    if start >= end:
        return
    k = array[start]
    left_flag = start
    right_flag = end
    while left_flag < right_flag:
        while left_flag < right_flag and array[right_flag] > k:
            right_flag -=1
        array[left_flag], array[right_flag] = array[right_flag], array[left_flag]

        while left_flag < right_flag and array[left_flag] <= k:
            left_flag +=1
        array[left_flag], array[right_flag] = array[right_flag], array[left_flag]
        print(array,left_flag,right_flag)
    quick_sort(array,start,left_flag-1)
    quick_sort(array,left_flag+1,end)

if __name__ == '__main__':
    array = [65,34,7,6,11,8,7,9,0,44,32,45,6,7,333]
    time_start=time.time()
    quick_sort(array,0,len(array)-1)
    time_end=time.time()
    print(-time_start+time_end)