"""
Local node REST api.
"""
from __future__ import absolute_import

import os
import httplib

# pylint: disable=E0611,F0401
import flask
import flask.ext.restplus as restplus
from flask import request

from treadmill import webutils

def _archive_type():
    """Get archive type from query string."""
    archive_type = request.args.get('type')
    if not archive_type:
        archive_type = 'app'

    if archive_type not in ['app', 'sys']:
        # TODO: need to raise validation error exception.
        raise Exception('Invalid archive type.')

    return archive_type

# pylint: disable=W0232,R0912
def init(api, cors, impl):
    """Configures REST handlers for allocation resource."""

    namespace = api.namespace('local',
                              description='Local server REST operations')

    @namespace.route('/running',)
    class _RunningList(restplus.Resource):
        """Treadmill Allocation resource"""

        @webutils.get_api(api, cors)
        def get(self):
            """Returns list running containers."""
            return impl.running.list()

    @namespace.route('/running/<instance>',)
    class _RunningDetails(restplus.Resource):
        """Treadmill Allocation resource"""

        @webutils.get_api(api, cors)
        def get(self, instance):
            """Returns list running containers."""
            return impl.running.get(instance)

    @namespace.route('/running/<instance>/sys/<component>')
    class _RunningSystemLog(restplus.Resource):
        """System log stream."""

        def get(self, instance, component):
            """Return content of archived file.."""
            mimetype = 'text/plain'
            return flask.Response(
                impl.running.lines(instance, 'sys', component),
                mimetype=mimetype
            )

    @namespace.route('/running/<instance>/service/<component>')
    class _RunningServiceLog(restplus.Resource):
        """System log stream."""

        def get(self, instance, component):
            """Return content of archived file.."""
            mimetype = 'text/plain'
            return flask.Response(
                impl.running.lines(instance, 'services', component),
                mimetype=mimetype
            )

    @namespace.route('/archive/<instance>',)
    class _ArchiveList(restplus.Resource):
        """Archive resource."""

        @webutils.get_api(api, cors)
        def get(self, instance):
            """Returns list of archives."""
            archive_type = _archive_type()
            return impl.archive.list(archive_type, instance)

    @namespace.route('/archive/<instance>/<idx>',)
    class _ArchiveDetails(restplus.Resource):
        """Archive resource."""

        @webutils.get_api(api, cors)
        def get(self, instance, idx):
            """Returns archive details."""
            idx = int(idx)
            archive_type = _archive_type()
            return impl.archive.get(archive_type, instance, idx)

    @namespace.route('/archive/<instance>/<idx>/file/<path:path>')
    class _ArchiveFile(restplus.Resource):
        """Archive resource."""

        def get(self, instance, idx, path):
            """Return content of archived file.."""
            idx = int(idx)
            archive_type = _archive_type()
            mimetype = 'text/plain'

            return flask.Response(
                impl.archive.lines(archive_type, instance, idx, path),
                mimetype=mimetype
            )

    @namespace.route('/archive/<instance>/<idx>/fetch')
    class _ArchiveAsAttachment(restplus.Resource):
        """Download archive as attachment."""

        def get(self, instance, idx):
            """Return content of archived file.."""
            idx = int(idx)
            archive_type = _archive_type()
            fname = impl.archive.path(archive_type, instance, idx)

            if not os.path.exists(fname):
                return 'Not found.', httplib.NOT_FOUND

            return flask.send_file(
                fname,
                as_attachment=True,
                attachment_filename=os.path.basename(fname)
            )


#coding: utf-8

import statsd

class StatsdClient(object):

    def __init__(self, *args, **kwargs):
        self._client = None
        if args or kwargs:
            self._client = statsd.StatsClient(*args, **kwargs)

    def init_app(self, app):
        cfg = app.config
        kwargs = dict(
            host=cfg.get('STATSD_HOST', 'localhost'),
            port=cfg.get('STATSD_PORT', 8125),
            prefix=cfg.get('STATSD_PREFIX', None),
            maxudpsize=cfg.get('STATSD_MAXUDPSIZE', 512),
            ipv6=cfg.get('STATSD_IPV6', False)
        )
        self._client = statsd.StatsClient(**kwargs)

    def __getattribute__(self, name):
        if name not in ['init_app', '_client']:
            return getattr(self._client, name)
        else:
            return super(StatsdClient, self).__getattribute__(name)


#初始化接口#
from WindPy import *
import json
w.start();

#获取上交所A股代码#
AllShAStock = w.wset("SectorConstituent","date=20170502;sectorId=a001010200000000;field=wind_code");
if AllShAStock.ErrorCode != 0:
        print("Get Data failed! exit!")
        exit()
#获取深圳交所A股代码#
AllSzAStock = w.wset("SectorConstituent","date=20170502;sectorId=a001010300000000;field=wind_code");
if AllSzAStock.ErrorCode != 0:
        print("Get Data failed! exit!")
        exit()
#获取所有A股代码#
AllAStock =w.wset("SectorConstituent","date=20170502;sectorId=a001010100000000;field=wind_code");
if AllAStock.ErrorCode != 0:
    print("Get Data failed! exit!")
    exit()
#把股票代码转变为列表格式
stock_code=AllAStock.Data[0]
stock_code
len(stock_code)


import random

from marlgrid import envs
import gym
import numpy as np
from marlgrid.envs import env_from_config

# from marlgrid.envs.empty import EmptyColorMultiGrid
from marlgrid.envs.hallways import HallWaysMultiGrid

def main():
    env = gym.make(
        "MarlGrid-2AgentComms15x15-v0"
    )  # ('MarlGrid-2AgentEmptyColor15x15-v0')

    env.reset()

    obs_list = env.reset()
    env.render()
    done = False
    steps = 0
    # actions = [[2,2],[2,2],[2,2],[2,2],[2,2],[2,2],[0,5],[5,2],
    #            [2,2],[2,2],[2,2],[2,2],[2,2],[2,2],[2,2],[0,0],
    #            [2,2],[2,2],[2,2],[2,2],[2,2],[5,2],[2,5],[5,5]]
    # actions = [[0,0],[0,0],[2,0], [0,0], [2,0], [0,0],[0,2],[2,0]]
    actions = [
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 0],
        [0, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 0],
    ]

    actions = [
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [1, 1],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [2, 2],
        [0, 0],
        [2, 2],
        [2, 0],
    ]
    # while not done:
    for action in actions:
        # OPTIONAL: render the whole scene + birds eye view
        # agent_actions = np.random.randint(env.action_space[0].n, size=len(env.action_space))
        next_obs_list, rew_list, done, _ = env.step(action)
        steps += 1
        env.render()
        print(steps, action, rew_list, done)
        print()

def main2():
    env = gym.make(
        "MarlGrid-2AgentComms15x15-v0"
    )  # 'MarlGrid-2AgentComms15x15-v0')  # ('MarlGrid-2AgentEmptyColor15x15-v0')
    env = gym.wrappers.Monitor(
        env,
        "./recording",
        video_callable=lambda episode_id: episode_id == 0,
        force=True,
    )
    env.seed(55)
    env.reset()
    for _ in range(100):
        env.render()
        action = env.action_space.sample()
        obs, reward, done, _ = env.step(action)

    print("wait")

def main3():
    env_config = {
        "env_class": "HallWaysMultiGrid",  ##"EmptyColorMultiGrid",
        "grid_size": 15,
        "goal_colors": ["blue", "red", "blue", "red"],
        "goal_coordinates": [[7, 1], [8, 1], [8, 13], [7, 13]],
        "max_steps": 250,
        "respawn": False,
        "ghost_mode": False,
        "reward_decay": False,
    }
    player_interface_config = {
        "view_size": 7,
        "view_offset": 1,
        "view_tile_size": 1,
        "observation_style": "rich",
        "see_through_walls": False,
        "observe_position": True,
        "see_color_in_view_bool": True,
        "observe_orientation": True,
        "color": "orange",
    }
    p1 = dict(player_interface_config.copy())
    p2 = dict(player_interface_config.copy())
    p1["color"] = "orange"
    p2["color"] = "yellow"
    env_config["agents"] = [p1, p2]
    env = env_from_config(env_config)
    obs = env.reset()
    env.render()
    print(obs[0])

def main4():
    # env = gym.make("MarlGrid-2AgentCommGame15x15-v0")
    env_config = {
        "env_class": "CommunicationGameEnv",
        "grid_size": 15,
        "respawn": False,
        "ghost_mode": False,
        "reward_decay": False,
        "block_coordinates": [(1, 1), (13, 1), (1, 13), (13, 13)],
        "block_colors": ["blue", "red", "cyan", "pink"],
        "comm_blocks_coordinates": [(7, 4), (7, 10)],
        "max_steps": 250,
    }
    player_interface_config = {
        "view_size": 7,
        "view_offset": 1,
        "view_tile_size": 1,
        "observation_style": "rich",
        "see_through_walls": False,
        "observe_position": True,
        "see_color_in_view_bool": True,
        "observe_orientation": True,
        "color": "green",
    }
    p1 = dict(player_interface_config.copy())
    p2 = dict(player_interface_config.copy())
    # p1["color"] = "orange"
    # p2["color"] = "yellow"
    env_config["agents"] = [p1, p2]
    env = env_from_config(env_config)

    seed = 22
    np.random.seed(seed)
    random.seed(seed)
    env.seed(seed)
    env.reset()
    env.render()

    actions = [
        (0, 2),
        (2, 2),
        (2, 2),
        (2, 2),
        (2, 2),
        (0, 1),
        (2, 2),
        (2, 2),
        (1, 2),
        (1, 2),
        (1, 2),
    ]
    actions2 = [
        (0, 1),
        (2, 2),
        (2, 2),
        (2, 1),
        (2, 2),
        (1, 0),
        (2, 2),
        (2, 2),
        (2, 2),
        (2, 1),
        (2, 2),
        (2, 2),
        (2, 2),
        (2, 2),
        (2, 2),
        (2, 2),
    ]
    print("starting")
    for action in actions2:
        # action = env.action_space.sample()
        obs, reward, done, _ = env.step(action)
        print()
        env.render()
        print(
            "obs:",
            obs[0]["see_color_in_view"],
            obs[1]["see_color_in_view"],
            "reward",
            reward,
            "done",
            done,
        )
        if done:
            break

if __name__ == "__main__":
    main4()


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 30 11:55:29 2020
获取申万行业分类数据并更新入库
@author: 李博
"""
import pandas as pd
import json
import datetime
import time
from starlette.requests import Request
#from fastapi import FastAPI
from fastapi import APIRouter
router = APIRouter()
from starlette.templating import Jinja2Templates
import tushare as ts
pro = ts.pro_api('78282dabb315ee578fb73a9b328f493026e97d5af709acb331b7b348')
# MONGODB CONNECT
#入库
from pymongo import MongoClient
#client = MongoClient('mongodb://112.12.60.2:27017')
client = MongoClient('mongodb://127.0.0.1:27017')
mydb=client["ptest"]

today=time.strftime('%Y%m%d',)
#计算指定日期的前N天的时间戳
def get_day_time(n):
    #the_date = datetime.datetime(2018,11,10) #指定当前日期 2018-11-10
    the_date = datetime.datetime.now()
    #the_date_str = the_date.strftime('%Y%m%d')
    pre_date = the_date - datetime.timedelta(days=n)
    pre_date_str = pre_date.strftime('%Y%m%d')#将日期转换为指定的显示格式
    return pre_date_str

#stocks=getstocks('801740.SI')
#print (stocks)

#获取当前分类的成份股详情
    '''
def getstocksdetail(indexcode,stockbasiccollection):
    stocks_items=[]
    stocks = pro.index_member(index_code=indexcode)
    stockslist = stocks['con_code'].tolist()
    for i in stockslist:
        print (i)
        stock_code = i
        query = { "ts_code": stock_code}
        stock_basic = stockbasiccollection.find(query)[0]
        stockname = stock_basic['name']
        area = stock_basic['area']
        industry = stock_basic['industry']
        #拼装json
        data = {"stockcode":stock_code,"stockname":stockname,"area":area,"industry":industry}
        print (data)
        stocks_items.append(data)
    return stocks_items
#stocks=getstocksdetail('801740.SI')
#print (stocks)
'''

#获取申万一到三级行业列表
def get_sw_list(src,level):
    df = pro.index_classify(level=level, src=src)
    records = json.loads(df.T.to_json()).values()
    mycollection=mydb[src+level]
    mycollection.remove()
    mycollection.insert_many(records)
    print (src,level,str(len(df)))
    return df

#获取基本信息
df_stockbasic = pro.stock_basic(exchange='', list_status='L', fields='ts_code,market,name,area,industry,list_date')
df_stockbasic['stockname'] = df_stockbasic['name']
#获取每日行情
df_tradedate = pro.trade_cal(exchange='SSE', is_open='1',fileds='cal_date',start_date=get_day_time(10), end_date=get_day_time(0))
lasttradeday = df_tradedate['cal_date'].tail(1).iloc[0]
df_daily = pro.daily(trade_date=lasttradeday)
#print (df_daily)
if (df_daily.empty):
    print ('df empty')
    df_tradedate = pro.trade_cal(exchange='SSE', is_open='1',fileds='cal_date',start_date=get_day_time(10), end_date=get_day_time(1))
    lasttradeday = df_tradedate['cal_date'].tail(1).iloc[0]
    df_daily = pro.daily(trade_date=lasttradeday)
    #print ('get_df_dailyc',len(df_daily),lasttradeday)
df_daily['amount'] = df_daily['amount']/10000
#获取每日指标
df_tradedate = pro.trade_cal(exchange='SSE', is_open='1',fileds='cal_date',start_date=get_day_time(10), end_date=get_day_time(0))
lasttradeday = df_tradedate['cal_date'].tail(1).iloc[0]
df_daily_basic = pro.daily_basic(ts_code='', trade_date=lasttradeday, fields='ts_code,turnover_rate,volume_ratio,pe_ttm,pb,circ_mv')
if (df_daily_basic.empty):
    df_tradedate = pro.trade_cal(exchange='SSE', is_open='1',fileds='cal_date',start_date=get_day_time(10), end_date=get_day_time(1))
    lasttradeday = df_tradedate['cal_date'].tail(1).iloc[0]
    df_daily_basic = pro.daily_basic(ts_code='', trade_date=lasttradeday, fields='ts_code,turnover_rate,volume_ratio,pe_ttm,pb,circ_mv')
    #print ('get_df_daily_basic',len(df_daily_basic),lasttradeday)
df_daily_basic['circ_mv'] = df_daily_basic['circ_mv']/10000
#获取当前分类的成份股
def get_sw_stocks(src,level):
    df = pro.index_classify(level=level, src=src)
    for index in df['index_code']:
        i = 0
        while i < 3:
            try:
                df_swstocks = pro.index_member(index_code=index)
                break
            except:
                print ('PAUSE WAITING 60S TO RECONNECTION')
                time.sleep(60)
                i += 1
        #df_swstocks = pro.index_member(index_code=index)
        df_swstocks['ts_code'] = df_swstocks['con_code']
        df_temp = pd.merge(df_swstocks, df_stockbasic, how='left', on='ts_code')
        df = pd.merge(df_temp, df_daily_basic, how='left', on='ts_code')
        df2 = pd.merge(df, df_daily, how='left', on='ts_code')
        #df.to_csv('./data/sw/'+i+'_'+src+level+'.csv')
        mycollection=mydb['stocks_'+index]
        mycollection.remove()
        records = json.loads(df2.T.to_json()).values()
        mycollection.insert_many(records)
        print (src+level+index)

#标准写入函数
def csvtomongodb(filename,collection):
   #client = MongoClient('mongodb://112.12.60.2:27017')
   client = MongoClient('mongodb://127.0.0.1:27017')
   mydb=client["ptest"]
   mycollection=mydb[collection]
   mycollection.remove()
   path_df=open('./data/sw/'+filename+'.csv','r',encoding='UTF-8')
   df_csv = pd.read_csv(path_df)
   records = json.loads(df_csv.T.to_json()).values()
   #print (records)
   mycollection.insert(records)

#入库
def get_sw_stocks_data():
    get_sw_list('sw','l1')
    get_sw_list('sw','l2')
    get_sw_list('sw','l3')
    get_sw_stocks('sw','l1')
    get_sw_stocks('sw','l2')
    get_sw_stocks('sw','l3')

#申万行业分类数据入库
def get_sw_level_data():
    get_sw_list('sw','l1')
    get_sw_list('sw','l2')
    get_sw_list('sw','l3')

tmp = Jinja2Templates(directory='./api/templates')
@router.get('/update/sw/')
async def get_indexs(request:Request):
    get_sw_list('sw','l1')
    get_sw_list('sw','l2')
    get_sw_list('sw','l3')
    get_sw_stocks('sw','l1')
    get_sw_stocks('sw','l2')
    get_sw_stocks('sw','l3')
    return tmp.TemplateResponse('update_data.html',
                                {'request':request
                                 })


from rest_framework.views import APIView
from rest_framework.response import Response
from users.models import *
from employers.models import *
from category.models import *
from skill.models import Skill
from packages.globalfunction import *
from .serializers import *
from users.serializers import *
from skill.serializers import SkillSerializer
import json
from applicants.models import Applicant

from rest_framework import status, generics, permissions
from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope, TokenHasScope

from rest_framework.parsers import MultiPartParser, FormParser

# -------------------- edit profile --------------------------

class editProfile(APIView):
	'''
		Parameters
			user_id : required \n

	'''
	parser_classes = (MultiPartParser, FormParser)
	def post(self,request):
		permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]

		if "user_id" in request.POST and request.POST["user_id"] !="":
			user_id = int(request.POST["user_id"])
		else:
			return Response(responsejson("Please enter user Id", status=422) )

		try :
			empObj = Employer.objects.get(user_id=user_id)
		except Exception as e:
			return Response(responsejson("This user does not exist.", status=422) )

		# Get all post method data
		userData = managePostData(request.POST)

		if "logo_img" in request.FILES and request.FILES["logo_img"] !="":
			fileName = ImageSaveByFile(request,'images','logo_img')
			if fileName !='':
				userData['logo_img'] = fileName

		# print(userData)

		if "employer_name" in userData and userData["employer_name"] !="":
			userData['name'] = userData["employer_name"]
			del userData['employer_name']

		try:
			update_profile_serialize = updateProfileSerializer(data=userData)
			if update_profile_serialize.is_valid():
				Employer.objects.filter(user_id=user_id).update(**userData)
			else:
				return Response(responsejson(update_profile_serialize.errors,status=422))

			# return data
			userDetails = setUserSessionValue(user_id,'employer')
			# userObj = User.objects.get(id=user_id)
			# userSerialize = UserSerializer(userObj)
			# userDetails = makeBlank(userSerialize.data.items())

			# empObj = Employer.objects.get(user_id=user_id)
			# empSerializer = EmployerLoginSerializer(empObj)

			# userDetails.update(makeBlank(empSerializer.data.items()))
			# userDetails['role'] = 'employer'

			return Response(responsejson(userDetails,status=200))
		except Exception as e:
			print(e)
			return Response(responsejson("Something goes wrong with your data, Please try again.",status=422))

# ------------------------------------ Add new job ---------------------------------

class addJob(APIView):
	'''
		Parameters
			user_id : required \n
			industry : required \n
			functional_area : required \n
			positions : required \n
			location : required \n
			responsibilities : required \n
			desired_skills : required \n
			min_vage : required \n
			salary : required \n
			max_applicants : required \n
	'''
	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]
	def post(self,request):
		errorDict = isUserExist(request)

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		# Get all post method data
		jobData = managePostData(request.POST)
		user_id = jobData['user_id']
		jobData['user'] = int(user_id)
		del jobData['user_id']

		skillsId = []
		if "newSkills" in jobData and jobData['newSkills'] != '':
			skillsId = addNewSkills(jobData['newSkills'])
			del jobData['newSkills']

		if "existSkills" in jobData and jobData['existSkills'] != '':
			existSkills = jobData['existSkills']
			existSkills = existSkills.split(",")
			skillsId = list(map(str, skillsId))
			skillsId = skillsId + existSkills
			del jobData['existSkills']

		skillDict = {"skill":skillsId}
		jobData["skills"] = skillDict
		# jobData["skills"] = json.dumps(skillDict)

		try:
			job_serializer = AddJobSerializer(data=jobData)

			if job_serializer.is_valid():
				job_serializer.save()
				return Response(responsejson("Job created successfully", status=200) )
			else:
				return Response(responsejson(job_serializer.errors,status=422))

		except Exception as e:
			print(e)
			errorDict['error'] = "Something goes wrong Please try again"
			return Response(responsejson(errorDict, status=422) )

# ------------------------------------------------------- Job listing api ----------------------------------------------------

class jobList(APIView):
	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]

	def post(self,request):
		errorDict = isUserExist(request)
		if errorDict:
			return Response(responsejson(errorDict,status=422))

		postData = managePostData(request.POST)
		user_id = postData['user_id']

		try:
			jobs = Job.objects.filter(user_id=user_id).order_by('-created_on')
			jobDict = []
			for job in jobs:
				singleJob = JobListingSerializer(job)
				singleJob = makeBlank(singleJob.data.items())
				jobDict.append(singleJob)

			return Response(responsejson(jobDict,status=200))
		except Exception as e:
			print(e)
			errorDict['error'] = "There are no jobs"
			return Response(responsejson(errorDict,status=422))

# ------------------------------------------------------- Job Delete ----------------------------------------------------

class deleteJob(APIView):
	'''
		Parameters
			user_id : required \n
			job_id : required \n
	'''

	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]
	def post(self,request):

		errorDict = isUserExist(request)
		errorDict = fieldValidation(request,'job_id','Please enter job_id',errorDict)

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		postData = managePostData(request.POST)
		try:
			jobObj = Job.objects.get(id=postData['job_id'],user_id=postData['user_id'])
			jobObj.delete()
			return Response(responsejson("Job Deleted successfully",status=200))

		except Exception as e:
			print(e)
			errorDict['error'] = "This job is not created by that user"
			return Response(responsejson(errorDict, status=422) )

# ----------------------------------------------------------- Get Job Detail ----------------------------------------------------

class getJobDetail(APIView):
	'''
		Parameters
			user_id : required \n
			job_id : required \n
	'''

	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]
	def post(self,request):
		errorDict = isUserExist(request)
		errorDict = fieldValidation(request,'job_id','Please enter job_id',errorDict)

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		postData = managePostData(request.POST)

		user_id = postData['user_id']
		job_id = postData['job_id']
		try:
			jobObj = Job.objects.get(id=job_id)

			getJob = JobListingSerializer(jobObj)
			getJob = makeBlank(getJob.data.items())

			empObj = Employer.objects.get(user_id=getJob['user_id'])
			serializerData = EmployerLoginSerializer(empObj)
			getJob['user_details'] = makeBlank(serializerData.data.items())

			if len(getJob['skills']) > 0:
				skillsIdData = getJob['skills']
				skillsIdData = [int(i) for i in skillsIdData['skill']]
				selectedSkillData = Skill.objects.filter(id__in=skillsIdData).only('id','name')

				skillDict = []
				for skill in selectedSkillData:
					singleSkill = SkillSerializer(skill)
					singleSkill = makeBlank(singleSkill.data.items())
					skillDict.append(singleSkill)

				getJob['skills'] = skillDict

			return Response(responsejson(getJob,status=200))

		except Exception as e:
			print(e)
			errorDict['error'] = "This job is not created by that user"
			return Response(responsejson(errorDict, status=422) )

# -------------------------------------------- Edit Job ----------------------------------------------

class editJob(APIView):
	'''
		Parameters
			user_id : required \n
			job_id : required \n
			industry : required \n
			functional_area : required \n
			positions : required \n
			location : required \n
			responsibilities : required \n
			desired_skills : required \n
			min_vage : required \n
			salary : required \n
			max_applicants : required \n
	'''
	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]
	def post(self,request):
		errorDict = isUserExist(request)
		user_id = request.POST["user_id"]

		if "job_id" in request.POST and request.POST["job_id"] !="":
			job_id = request.POST["job_id"]
			try :
				jobObj = Job.objects.get(id=job_id,user_id=user_id)
			except Exception as e:
				errorDict['error'] = "This Job does not exist."
		else:
			errorDict['job_id'] = "Please enter Job Id"

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		if "title" in request.POST and request.POST["title"] !="":
			jobObj.title = request.POST["title"]
		else:
			errorDict['title'] = "Please enter Title"

		if "industry" in request.POST and request.POST["industry"] !="":
			industry = request.POST["industry"]
			try :
				cateObj = Categories.objects.get(id=int(industry))
				jobObj.industry = cateObj
			except Exception as e:
				errorDict['industry'] = "This industry-category does not exist."

		else:
			errorDict['industry'] = "Please enter industry"

		if "functional_area" in request.POST and request.POST["functional_area"] !="":
			functional_area = request.POST["functional_area"]

			try :
				functObj = Categories.objects.get(id=functional_area)
				jobObj.functional_area = functObj
			except Exception as e:
				errorDict['functional_area'] = "This functional-category does not exist."

		else:
			errorDict['functional_area'] = "Please enter functional area"

		if "positions" in request.POST and request.POST["positions"] !="":
			jobObj.positions = request.POST["positions"]
		else:
			errorDict['positions'] = "Please enter Positions"

		# if "location" in request.POST and request.POST["location"] !="":
		# 	jobObj.location = request.POST["location"]
		# else:
		# 	errorDict['location'] = "Please enter Location"

		if "responsibilities" in request.POST and request.POST["responsibilities"] !="":
			jobObj.responsibilities = request.POST["responsibilities"]
		else:
			errorDict['responsibilities'] = "Please enter responsibility"

		# if "desired_skills" in request.POST and request.POST["desired_skills"] !="":
		# 	jobObj.desired_skills = request.POST["desired_skills"]
		# else:
		# 	errorDict['desired_skills'] = "Please enter responsibility"

		if "min_vage" in request.POST and request.POST["min_vage"] !="":
			jobObj.min_vage = request.POST["min_vage"]
		else:
			errorDict['min_vage'] = "Please enter minimum vages"

		if "min_exp" in request.POST and request.POST["min_exp"] !="":
			jobObj.min_exp = request.POST["min_exp"]
		else:
			errorDict['min_exp'] = "Please enter minimum experience"

		if "max_applicants" in request.POST and request.POST["max_applicants"] !="":
			jobObj.max_applicants = request.POST["max_applicants"]
		else:
			errorDict['max_applicants'] = "Please enter maximum applicants"

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		skillsId = []
		if "newSkills" in request.POST and request.POST['newSkills'] != '':
			skillsId = addNewSkills(request.POST['newSkills'])
			del request.POST['newSkills']

		if "existSkills" in request.POST and request.POST['existSkills'] != '':
			existSkills = request.POST['existSkills']
			existSkills = existSkills.split(",")
			skillsId = list(map(str, skillsId))
			skillsId = skillsId + existSkills
			# del request.POST['existSkills']

		skillDict = {"skill":skillsId}
		jobObj.skills = skillDict
		# jobData["skills"] = skillDict

		try:
			jobObj.save()
			return Response(responsejson("Job Update successfully", status=200) )
		except Exception as e:
			print(e)
			errorDict['error'] = "This job is not created by that user"
			return Response(responsejson(errorDict, status=422) )

# -------------------------------------------- Update Job Status ----------------------------------------------

class updateJobStatus(APIView):
	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]

	def post(self,request):
		if "user_id" in request.POST and request.POST["user_id"] !="":
			user_id = request.POST["user_id"]

			try :
				userObj = User.objects.get(id=user_id)
			except Exception as e:
				return Response(responsejson("This user does not exist.", status=422) )

		else:
			return Response(responsejson("Please enter user Id", status=422) )

		if "job_id" in request.POST and request.POST["job_id"] !="":
			job_id = request.POST["job_id"]
		else:
			return Response(responsejson("Please enter job Id", status=422) )

		try:
			jobObj = Job.objects.get(id=job_id,user_id=user_id)
			if jobObj.status == 1:
				jobObj.status = 0
			else:
				jobObj.status = 1

			jobObj.save()
			return Response(responsejson("Job Status Update Successfully",status=200))

		except Exception as e:
			print(e)
			return Response(responsejson("This job is not created by that user", status=422) )

# --------------------------------------- Get Job applicants List ---------------------------------------

class getApplicantList(APIView):
	permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope]

	def post(self,request,*args, **kwargs):

		from applicants.serializers import ApplicantListSerializer
		from jobseekers.models import Resumes,JobSeeker
		from jobseekers.serializers import ResumeDetailsSerializer,JobseekerLoginSerializer

		errorDict = isUserExist(request)
		errorDict = fieldValidation(request,'job_id','Please enter job_id',errorDict)
		if errorDict:
			return Response(responsejson(errorDict,status=422))

		errorDict = {}
		postData = managePostData(request.POST)

		try:
			applicants = Applicant.objects.filter(job=int(postData['job_id']))
			if not applicants:
				return Response(responsejson([], status=200) )

			applicantDict = []
			for applicant in applicants:
				# applicant deatails
				singleApplicant = {}
				singleApplicant = ApplicantListSerializer(applicant)
				singleApplicant = makeBlank(singleApplicant.data.items())

				# Fetch user name from jobseeker table
				jobseekerData = {}
				jobseekerData = JobSeeker.objects.get(user_id=applicant.user_id)
				jobseekerData = JobseekerLoginSerializer(jobseekerData)
				jobseekerData = makeBlank(jobseekerData.data.items())

				singleApplicant.update(jobseekerData)

				# fetch data from resume table
				try:
					getResumeData = Resumes.objects.get(user_id=applicant.user_id)
					getResumeData = ResumeDetailsSerializer(getResumeData)
					getResumeData = makeBlank(getResumeData.data.items())
					singleApplicant.update(getResumeData);
				except Exception as e:
					print(e)

				applicantDict.append(singleApplicant)
			return Response(responsejson(applicantDict, status=200) )
		except Exception as e:
			print(e)
			return Response(responsejson("This job is not created.", status=422) )

# --------------------------------------- Update Applicant status ---------------------------------------
class updateApplicantStatus(APIView):
	def post(self,request):
		errorDict = isUserExist(request)

		postData = managePostData(request.POST)

		if 'action' not in postData or postData["action"] =="":
			errorDict['action'] = "Action field is required"

		if 'applicant_id' not in postData or postData["applicant_id"] =="":
			errorDict['applicant_id'] = "Applicant field is required"

		if errorDict:
			return Response(responsejson(errorDict,status=422))

		try:
			for id in postData['applicant_id'].split(','):
				updateData = {}
				updateData['status'] = int(postData['action'])
				Applicant.objects.filter(id=int(id)).update(**updateData)

			return Response(responsejson("status update successfully",status=200))
		except Exception as e:
			print(e)
			return Response(responsejson("Something went wrong, Please try again",status=422))

		return Response(responsejson("Something went wrong",status=422))


import pickle
import os

import pandas as pd
import numpy as np

from shared_modeling_simulation import get_alien_paths
from shared_aliens import alien_initial_Q, update_Qs_sim
from AlienTask import Task

# Switches for this script
model_name = "soft"
verbose = False
n_subj = 31
n_sim_per_subj = 1
start_id = 0
param_names = np.array(['alpha', 'beta', 'forget', 'alpha_high', 'beta_high', 'forget_high'])
fake_data = False
model_to_be_simulated = "specify"  # "MSE"  # "MCMC" "specify"
# model_name = "/AliensMSEFitting/18-10-14/f_['alpha' 'beta' 'forget']_[[ 1 10  1]]_2018_10_14_9_47"  # 'Aliens/max_abf_2018_10_10_18_7_humans_n_samples10'  #
# Get save path
save_dir = get_alien_paths(False)['simulations']
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

# Get parameters
parameters = pd.DataFrame(columns=np.append(param_names, ['sID']))
parameter_dir = get_alien_paths(run_on_cluster=False)['fitting results']

if model_to_be_simulated == 'specify':

    parameters['alpha'] = 0.01 + 0.05 * np.random.rand(n_subj)  # 0 < alpha < 0.2
    parameters['beta'] = 6.5 + np.random.rand(n_subj)  # 1 < beta < 2
    parameters['forget'] = 0.005 + 0.01 * np.random.rand(n_subj)  # 0 < forget < 0.1

    parameters['alpha_high'] = 0.3 + 0.1 * np.random.rand(n_subj)  # 0 < alpha_high < 0.2
    parameters['beta_high'] = 4.5 + np.random.rand(n_subj)  # 1 < beta < 2
    parameters['forget_high'] = 0.2 + 0.1 * np.random.rand(n_subj)

    parameters['sID'] = range(n_subj)

# Load fitted parameters
elif model_to_be_simulated == 'MSE':

    parameters = pd.read_csv(parameter_dir + '/ten_best_{}.csv'.format(model_name), index_col=0)
    n_subj = min(n_subj, parameters.shape[0])
    parameters = parameters[:n_subj]
    parameters['sID'] = range(n_subj)

    if 'alpha_high' not in parameters:
        parameters['alpha_high'] = parameters['alpha'].copy()
    if 'beta_high' not in parameters:
        parameters['beta_high'] = parameters['beta'].copy()
    if 'forget_high' not in parameters:
        parameters['forget_high'] = parameters['forget'].copy()

elif model_to_be_simulated == 'MCMC':

    print('Loading {0}{1}.\n'.format(parameter_dir, model_name))
    with open(parameter_dir + model_name + '.pickle', 'rb') as handle:
        data = pickle.load(handle)
        model_summary = data['summary']
        model = data['model']

    model_summary = pd.read_csv(parameter_dir + model_name + '_summary.csv', index_col=0)
    for param_name in param_names:
        param_idx = [idx for idx in model_summary.index if param_name + '__' in idx]
        parameters[param_name] = model_summary.loc[param_idx[:n_subj], 'mean'].values
    parameters['sID'] = range(n_subj)
    parameters = pd.DataFrame(np.tile(parameters, (n_sim_per_subj, 1)))
    parameters = pd.DataFrame(np.array(parameters, dtype=float))
    parameters.columns = np.append(param_names, ['sID'])

if verbose:
    print("Parameters: {}".format(parameters.round(3)))

# Parameter shapes
n_sim = n_subj * n_sim_per_subj
beta_shape = (n_sim, 1)  # Q_low_sub.shape -> [n_subj, n_actions]
beta_high_shape = (n_sim, 1)  # Q_high_sub.shape -> [n_subj, n_TS]
forget_high_shape = (n_sim, 1, 1)  # -> [n_subj, n_seasons, n_TS]
forget_shape = (n_sim, 1, 1, 1)  # Q_low[0].shape -> [n_subj, n_TS, n_aliens, n_actions]

# Get numbers of things
n_seasons, n_aliens, n_actions = 3, 4, 3
n_TS = n_seasons
n_alien_repetitions = np.array([13, 7, 7])  # InitialLearning, Refresher2, Refresher3
n_season_repetitions = np.array([3, 2, 2])  # InitialLearning, Refresher2, Refresher3

# Initialize task
task = Task(n_subj)
n_trials, _, _ = task.get_trial_sequence("C:/Users/maria/MEGAsync/Berkeley/TaskSets/Data/version3.1/",
                                         n_subj, n_sim_per_subj, range(n_subj), fake_data)
print("n_trials", n_trials)

# For saving data
seasons = np.zeros([n_trials, n_sim], dtype=int)
TSs = np.zeros([n_trials, n_sim], dtype=int)
aliens = np.zeros([n_trials, n_sim], dtype=int)
actions = np.zeros([n_trials, n_sim], dtype=int)
rewards = np.zeros([n_trials, n_sim])
corrects = np.zeros([n_trials, n_sim])
p_lows = np.zeros([n_trials, n_sim, n_actions])
# Q_lows = np.zeros([n_trials, n_sim, n_TS, n_aliens, n_actions])
Q_highs = np.zeros([n_trials, n_sim])

print('Simulating {0} {2} agents on {1} trials.\n'.format(n_sim, n_trials, model_name))

Q_low = alien_initial_Q * np.ones([n_sim, n_TS, n_aliens, n_actions])
Q_high = alien_initial_Q * np.ones([n_sim, n_seasons, n_TS])

# Bring parameters into the right shape
alpha = parameters['alpha'].values
beta = parameters['beta'].values.reshape(beta_shape)
forget = parameters['forget'].values.reshape(forget_shape)
alpha_high = parameters['alpha_high'].values
beta_high = parameters['beta_high'].values.reshape(beta_high_shape)
forget_high = parameters['forget_high'].values.reshape(forget_high_shape)

for trial in range(np.sum(n_trials)):

    # Observe stimuli
    season, alien = task.present_stimulus(trial)

    # Print info
    if verbose:
        print("\n\tTRIAL {0}".format(trial))
        print("season:", season)
        print("alien:", alien)

    # Update Q-values
    [Q_low, Q_high, TS, action, correct, reward, p_low] =\
        update_Qs_sim(season, alien,
                           Q_low, Q_high,
                           beta, beta_high, alpha, alpha_high, forget, forget_high,
                           n_sim, n_actions, n_TS, task, verbose=verbose)

    # Store trial data
    seasons[trial] = season
    TSs[trial] = TS
    aliens[trial] = alien
    actions[trial] = action
    rewards[trial] = reward
    corrects[trial] = correct
    # Q_lows[trial] = Q_low
    Q_highs[trial] = Q_high[np.arange(n_sim), season, TS]
    p_lows[trial] = p_low

# Save data
for sID in range(n_sim):

    agent_ID = sID + start_id
    # Create pandas DataFrame
    subj_data = pd.DataFrame()
    subj_data["context"] = seasons[:, sID]
    subj_data["TS_chosen"] = TSs[:, sID]
    subj_data["sad_alien"] = aliens[:, sID]
    subj_data["item_chosen"] = actions[:, sID]
    subj_data["reward"] = rewards[:, sID]
    subj_data["correct"] = corrects[:, sID]
    subj_data["trial_type"] = "feed-aliens"
    subj_data["phase"] = np.array(task.phase).astype(str)
    subj_data["trial_index"] = np.arange(n_trials)
    # subj_data["p_low"] = p_norms[:, sID]
    subj_data["sID"] = agent_ID
    subj_data["block.type"] = "normal"
    subj_data["model_name"] = model_name
    for param_name in param_names:
        subj_data[param_name] = np.array(parameters.loc[sID, param_name])
    # subj_data["alpha"], subj_data["beta"], subj_data["forget"] = alpha[sID], beta.flatten()[sID], forget.flatten()[sID]
    # subj_data["alpha_high"], subj_data["beta_high"], subj_data["forget_high"] = alpha_high[sID], beta_high.flatten()[sID], forget_high.flatten()[sID]
    # subj_data["Q_low"] = Q_lows[:, sID]
    subj_data["Q_TS"] = Q_highs[:, sID]

    # Save to disc
    file_name = save_dir + "aliens_" + model_name + '_' + str(agent_ID) + ".csv"
    print('Saving file {0}'.format(file_name))
    subj_data.to_csv(file_name)


'''
@Author: Matthew Shabet
@Date: 2020-09-11 18:00:00
@LastEditTime: 2020-09-11 18:30:00
@LastEditors: Please set LastEditors
@Description: In User Settings Edit
'''
import csv
import requests
import os
import json
from datetime import datetime

url = 'https://services5.arcgis.com/fsYDFeRKu1hELJJs/arcgis/rest/services/FOHM_Covid_19_FME_1/FeatureServer/0/query?f=json&where=Region%20%3C%3E%20%27dummy%27&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&outSR=102100&resultOffset=0&resultRecordCount=25&resultType=standard&cacheHint=true'

# Get the data
response = requests.get(url, headers={'Connection': 'close'})
data = json.loads(response.text)["features"]

# Create and open the CSV
mkfile_time = datetime.strftime(datetime.now(), '%Y%m%d%H%M')
folder_path = './data/Sweden/'+ mkfile_time + '/'
if not os.path.exists(folder_path):
    os.makedirs(folder_path)
file = open(folder_path+'table.csv', 'w', newline='', encoding='utf-8-sig')
writer = csv.writer(file)

# Write each line to the csv
headers = ["Region", "Cases", "Cases/100k pop.", "Deaths", "Intensive care"]
writer.writerow(headers)
for d in data:
	d = d["attributes"]
	row = [d["Region"], d["Totalt_antal_fall"], d["Fall_per_100000_inv"], d["Totalt_antal_avlidna"], d["Totalt_antal_intensivvårdade"]]
	writer.writerow(row)


#!/usr/bin/env/python

#name = fasta_orf_parser.py
#author = Amy Kistler
#date = 20210406

#usage = python fasta_orf_parser.py source_filename search_term

#notes:
#hacky script to pull all records from batch fasta file with search_term in defline
#an output file with "_search_Term.fasta" suffix is generated from this script.

#import modules for fastaSlicer.py
import sys
import os
from Bio import SeqIO

#pull infile and prefix for output file from user-entered cli
infile = sys.argv[1]
print("")
print(infile)
file_prefix = infile.split(".")[0]
print(file_prefix)

#pull search_term from the user-entered cli
search_term = sys.argv[2]
print(search_term)

# set up file for output
outfile = file_prefix + "_" + search_term + ".fasta"
print(outfile)
print("")
o = open(outfile, "w")

#parse fasta file, probe defline for search_term, write title and seq to file
for seq_record in SeqIO.parse(infile, 'fasta'):
    print(seq_record.description)
    if search_term in seq_record.description:
        print(search_term + ' found in defline!')
        seq=(str(seq_record.seq))
        title=">" + seq_record.description
        #print(len(seq_record))
        #print("")
        print(title)
        print(seq)
        print("")
        o.write(title)
        o.write("\n")
        o.write(seq)
        o.write("\n")
    else:
        print('no ' + search_term + ' in defline!')
        pass

o.close()


from opentrons import protocol_api
from opentrons_functions.magbeads import (
    remove_supernatant, bead_wash, transfer_elute)
from opentrons_functions.transfer import add_buffer

metadata = {'apiLevel': '2.5',
            'author': 'Jon Sanders'}

# Set to `True` to perform a short run, with brief pauses and only
# one column of samples
test_run = True

if test_run:
    pause_bind = 5*60
    pause_mag = 10*60
    pause_dry = 5*60
    pause_elute = 5*60

    # Limit columns
    cols = ['A8', 'A9']
else:
    pause_bind = 5*60
    pause_mag = 10*60
    pause_dry = 5*60
    pause_elute = 5*60

    # Limit columns
    cols = ['A1', 'A2', 'A3', 'A4', 'A5', 'A6',
            'A7', 'A8', 'A9', 'A10', 'A11', 'A12']

# bead aspiration flow rate
bead_flow = .25

# wash mix mutliplier
wash_mix = 5

tb1_cols = ['A1', 'A2']

blt_col = 'A3'

tsb_col = 'A4'

i5_col = 'A5'

# PCR MM columns
pcr_cols = ['A6', 'A7']

# Wash 1 (TWB) columns
twb_cols = ['A1', 'A2']

twb_fill = 12000

h2o_col = 'A3'

beads_col = 'A4'

# EtOH columns
eth_cols = ['A5', 'A6', 'A7']

eth_fill = 12000

def run(protocol: protocol_api.ProtocolContext):

    # ### HackFlex Illumina-compatible library prep protocol

    # ### Deck

    # 1. samples; libraries
    # 2. reagent reservoir
    # 3. reagent strip tubes
    # 4. 300 tips (wash); 200f tips (elute)
    # 5. 10f tips (samples)
    # 6. i7 primers
    # 7. 20 tips (reagents)
    # 8. 300 tips (reagents)
    # 9. 10f tips (primers)
    # 10. mag module
    # 11. waste
    # 12. trash

    # define custom labware for strip tubes block
    # reagent strip tubes:
    # 1: TB1 200 µL
    # 2: TB1 200 µL
    # 3: BLT 150 µL
    # 4: TSB 150 µL
    # 5: i5 primers 150 µL
    # 6: PCR MM 200 µL
    # 7: PCR MM 200 µL

    # buffer reservoirs:
    # 1: TWB (10 mL)
    # 2: TWB (10 mL)
    # 3: H2O (8 mL)
    # 4: beads (6 mL)
    # 5: 80% EtOH
    # 6: 80% EtOH
    # 7: 80% EtOH

    # ### Setup

    # define deck positions and labware

    # define hardware modules
    magblock = protocol.load_module('magnetic module gen2', 10)
    magblock.disengage()

    # tips
    tiprack_samples = protocol.load_labware('opentrons_96_filtertiprack_10ul',
                                            5)
    tiprack_buffers = protocol.load_labware('opentrons_96_tiprack_300ul',
                                            8)
    tiprack_wash = protocol.load_labware('opentrons_96_tiprack_300ul',
                                         4)
    tiprack_primers = protocol.load_labware('opentrons_96_filtertiprack_10ul',
                                            9)
    tiprack_reagents = protocol.load_labware('opentrons_96_tiprack_20ul',
                                             7)

    # reagents
    # should be new custom labware with strip tubes
    reagents = protocol.load_labware('tubeblockvwrpcrstriptube_96_wellplate_250ul',
                                     3, 'reagents')
    buffers = protocol.load_labware('nest_12_reservoir_15ml',
                                    2, 'wash buffers')
    waste = protocol.load_labware('nest_1_reservoir_195ml',
                                  11, 'liquid waste')

    # plates
    samples = protocol.load_labware('biorad_96_wellplate_200ul_pcr',
                                   1, 'samples')
    i7_primers = protocol.load_labware('biorad_96_wellplate_200ul_pcr',
                                       6, 'i7 primers')

    # load plate on magdeck
    # mag_plate = magblock.load_labware('vwr_96_wellplate_1000ul')
    mag_plate = magblock.load_labware('nest_96_wellplate_100ul_pcr_full_skirt')

    # initialize pipettes
    pipette_left = protocol.load_instrument('p300_multi',
                                            'left',
                                            tip_racks=[tiprack_buffers])
    pipette_right = protocol.load_instrument('p10_multi',
                                            'right',
                                            tip_racks=[tiprack_reagents])

    # TB1 wells
    tb1_wells = [reagents[x] for x in tb1_cols]

    # TWB wash wells
    twb_wells = [buffers[x] for x in twb_cols]

    # PCR MM wells
    pcr_wells = [reagents[x] for x in pcr_cols]

    # EtOH wells
    eth_wells = [buffers[x] for x in eth_cols]

    # DNA plate

    # Step 1: Tagmentation
    # Diluted BLT: 1 mL; 120 (150 µL) per tip
    # TB1: 2.4 mL; 300 (350 µL) per tip

    # add TB1.
    # buffer tips 1

    tb1_wells, tb1_remaining = add_buffer(pipette_left,
                                          tb1_wells,
                                          mag_plate,
                                          cols,
                                          25,
                                          200,
                                          tip=None,
                                          tip_vol=300,
                                          remaining=None,
                                          drop_tip=True,
                                          dead_vol=10)

    # add BLT
    # reagent tips 2

    # mix BLT first
    pipette_right.pick_up_tip()
    pipette_right.mix(10,
                      10,
                      reagents[blt_col])
    pipette_right.transfer(10,
                           reagents[blt_col],
                           [mag_plate[x] for x in cols],
                           mix_before=(2, 10),
                           new_tip='never')
    pipette_right.drop_tip()

    # add sample

    for col in cols:
        pipette_right.pick_up_tip(tiprack_samples[col])
        pipette_right.transfer(10,
                               samples[col],
                               mag_plate[col],
                               mix_after=(10, 10),
                               new_tip='never',
                               trash=False)
        pipette_right.drop_tip()

    # Prompt user to remove plate and run on thermocycler
    protocol.pause('Remove plate from magblock, seal, and run '
                   'program TAG on thermocycler. Then spin down, unseal, '
                   'and return to magblock.')

    # Step 2: Stop reaction
    # TSB: 1 mL; 120 (150 µL) per tip

    # add TSB to each sample.
    # Prompt user to remove plate and run on thermocycler

    ### Is this step going to cross-contaminate? Seems wasteful to take.
    ### new tip for each sample. z = -1 meant to help.

    # reagent tips 2
    pipette_right.transfer(10,
                           reagents[tsb_col],
                           [mag_plate[x].top(z=-1) for x in cols],
                           touch_tip=False,
                           new_tip='once')

    protocol.pause('Remove plate from magblock, seal, vortex, spin, and run '
                   'program PTC on thermocycler. Then spin down, unseal, '
                   'and return to magblock.')

    # Step 3: Cleanup
    # TWB: 20 mL; 1200 (1500 µL) per tip

    # Magnet wash 2X

    # bind for specified length of time

    protocol.comment('Binding beads to magnet.')
    magblock.engage()

    protocol.delay(seconds=pause_mag)

    # ### Do first wash: 100 µL TWB
    # buffer tips 2

    # ### Do first wash
    protocol.comment('Doing wash #1.')
    twb_remaining, twb_wells = bead_wash(
                                         # global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments
                                         twb_wells,
                                         twb_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments,
                                         resuspend_beads=False,
                                         super_vol=70,
                                         rate=bead_flow,
                                         wash_vol=100,
                                         drop_super_tip=False,
                                         touch_wash_tip=True,
                                         mix_n=wash_mix,
                                         mix_vol=90,
                                         wash_tip_vol=300,
                                         super_tip_vol=200,
                                         remaining=None,
                                         pause_s=pause_mag)

    # ### Do second wash: 100 µL TWB
    # buffer tips 3
    protocol.comment('Doing wash #2.')

    twb_remaining, twb_wells = bead_wash(
                                         # global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments
                                         twb_wells,
                                         twb_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments,
                                         resuspend_beads=False,
                                         super_vol=120,
                                         rate=bead_flow,
                                         wash_vol=100,
                                         drop_super_tip=False,
                                         touch_wash_tip=True,
                                         mix_n=wash_mix,
                                         mix_vol=90,
                                         wash_tip_vol=300,
                                         super_tip_vol=200,
                                         remaining=twb_remaining,
                                         pause_s=pause_mag)

    # remove supernatant
    remove_supernatant(pipette_left,
                       mag_plate,
                       cols,
                       tiprack_wash,
                       waste['A1'],
                       super_vol=120,
                       rate=bead_flow,
                       bottom_offset=.8,
                       drop_tip=False)

    magblock.disengage()
    # remove supernatant
    # Step 3: amplification
    # MM: 3 mL; 350 (400 µL) per tip
    # buffer tips 4

    # Prompt user to unseal i7 PCR plate

    protocol.pause('Now, unseal i7 PCR primer plate, and uncap PCR master'
                   ' mix and i5 primer strip tubes.')

    pcr_wells, pcr_remaining = add_buffer(pipette_left,
                                          pcr_wells,
                                          mag_plate,
                                          cols,
                                          30,
                                          200,
                                          tip=None,
                                          tip_vol=300,
                                          remaining=None,
                                          drop_tip=True,
                                          dead_vol=10)

    # plate: primers i5
    # reagent tips 3
    pipette_right.transfer(10,
                           reagents[i5_col],
                           [mag_plate[x].top(z=-1) for x in cols],
                           touch_tip=True,
                           new_tip='once')

    # plate: primers i7
    for col in cols:
        pipette_right.pick_up_tip(tiprack_primers[col])
        pipette_right.transfer(10,
                               i7_primers[col],
                               mag_plate[col],
                               touch_tip=True,
                               new_tip='never',
                               trash=False)
        pipette_right.drop_tip()

    # Prompt user to remove plate and run on thermocycler

    protocol.pause('Remove plate from magblock, seal, vortex, spin and run '
                   'amplification program on thermocycler. Click continue when'
                   ' done.')

    protocol.delay(seconds=2)
    # Step 4: Size selection
    # H2O: 72 µL per sample; 7.2 mL; 600 per tip
    # EtOH: 400 µL per sample; 5000 per tip
    # Beads: 6 mL; 720 µL per tip

    # Prompt user to remove plate and run on thermocycler

    protocol.pause('Replace tips in position {0}'
                   ' with a fresh box'.format(tiprack_wash.parent))

    protocol.delay(seconds=2)

    protocol.pause('Remove sample plate from position {0}, seal, and store. '
                   'Place a new, clean, 96-well Nest PCR plate in position'
                   ' {0}. Remove library plate from PCR machine and spin at '
                   '280 xg for one minute. Then unseal and return to'
                   ' magblock.'.format(samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()

    protocol.delay(seconds=pause_mag)

    # Add buffers for large-cut size selection to new plate
    protocol.comment('Preparing large-cut bead conditions in new plate.')

    # add 40 µL H2O
    # buffer tips 5
    pipette_left.distribute(40,
                            buffers[h2o_col],
                            [samples[x].top(z=-1) for x in cols],
                            touch_tip=True,
                            disposal_volume=10,
                            new_tip='once')

    # Add 45 µL SPRI beads
    # buffer tips 6
    pipette_left.pick_up_tip()
    pipette_left.mix(10, 200, buffers[beads_col])
    pipette_left.distribute(45,
                            buffers[beads_col],
                            [samples[x].top(z=-1) for x in cols],
                            mix_before=(2,40),
                            touch_tip=True,
                            disposal_volume=10,
                            new_tip='never')
    pipette_left.drop_tip()

    # Transfer 45 µL PCR supernatant to new plate
    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   45,
                   z_offset=0.5,
                   x_offset=1,
                   rate=0.25,
                   drop_tip=False,
                   mix_n=7,
                   mix_vol=100)

    protocol.pause('Remove and discard plate from mag block. '
                   'Move plate in position {0} to mag block, and replace '
                   'with a new, clean 96-well Nest PCR plate.'.format(
                    samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()
    protocol.delay(seconds=pause_mag)

    # Add buffers for small-cut size selection to new plate
    # Add 20 µL SPRI beads
    # buffer tips 7
    pipette_left.pick_up_tip()
    pipette_left.mix(10, 100, buffers[beads_col])
    pipette_left.distribute(20,
                            buffers[beads_col],
                            [samples[x] for x in cols],
                            mix_before=(2,15),
                            touch_tip=True,
                            new_tip='never')
    pipette_left.drop_tip()

    # Transfer 115 µL large-cut supernatant to new plate

    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   115,
                   z_offset=0.5,
                   x_offset=1,
                   rate=0.25,
                   drop_tip=False,
                   mix_n=7,
                   mix_vol=100)

    protocol.pause('Remove and discard plate from mag block. '
                   'Move plate in position {0} to mag block, and replace '
                   'with a new, clean 96-well **BioRad** PCR plate. '
                   'Also, fill Ethanol wells at this stage.'.format(
                    samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()
    protocol.delay(seconds=pause_mag)

    # ### Do first wash: 150 µL EtOH
    # buffer tips 8
    protocol.comment('Doing wash #1.')
    eth_remaining, eth_wells = bead_wash(# global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments,
                                         eth_wells,
                                         eth_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments
                                         resuspend_beads=False,
                                         wash_vol=150,
                                         super_vol=125,
                                         drop_super_tip=False,
                                         mix_n=0,
                                         mix_vol=140,
                                         remaining=None,
                                         pause_s=pause_mag)

    # ### Do first wash: 150 µL EtOH
    # buffer tips 9
    protocol.comment('Doing wash #2.')
    eth_remaining, eth_wells = bead_wash(# global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments,
                                         eth_wells,
                                         eth_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments
                                         resuspend_beads=False,
                                         wash_vol=150,
                                         super_vol=170,
                                         drop_super_tip=False,
                                         mix_n=0,
                                         mix_vol=140,
                                         remaining=eth_remaining,
                                         pause_s=pause_mag)

    # ### Dry
    protocol.comment('Removing wash and drying beads.')

    # This should:
    # - pick up tip in position 8
    # - pick up supernatant from magplate
    # - dispense in waste, position 11
    # - repeat
    # - trash tip
    # - leave magnet engaged

    # remove supernatant

    remove_supernatant(pipette_left,
                       mag_plate,
                       cols,
                       tiprack_wash,
                       waste['A1'],
                       super_vol=170,
                       rate=bead_flow,
                       bottom_offset=.3,
                       drop_tip=True)

    # dry

    protocol.delay(seconds=pause_dry)

    protocol.pause('Replace empty tiprack in position {0} with new rack of '
                   '200 µL filter tips.'.format(tiprack_wash.parent))

    # ### Elute
    protocol.comment('Eluting DNA from beads.')

    # This should:
    # - disengage magnet
    # - pick up tip from position 6
    # - pick up reagents from column 2 of position 9
    # - dispense into magplate
    # - mix 10 times
    # - blow out, touch tip
    # - return tip to position 6
    # - wait (5 seconds)
    # - engage magnet
    # - wait (5 seconds)
    # - pick up tip from position 6
    # - aspirate from magplate
    # - dispense to position 3
    # - trash tip

    # transfer elution buffer to mag plate

    magblock.disengage()

    # add elution buffer and mix
    for col in cols:
        pipette_left.pick_up_tip(tiprack_wash.wells_by_name()[col])
        pipette_left.aspirate(35, buffers[h2o_col], rate=1)
        pipette_left.dispense(35, mag_plate[col].bottom(z=1))
        pipette_left.mix(10, 29, mag_plate[col].bottom(z=1))
        pipette_left.blow_out(mag_plate[col].top())
        pipette_left.touch_tip()
        # we'll use these same tips for final transfer
        pipette_left.return_tip()

    protocol.delay(seconds=pause_elute)

    # bind to magnet
    protocol.comment('Binding beads to magnet.')

    magblock.engage()

    protocol.delay(seconds=pause_mag)

    protocol.comment('Transferring eluted DNA to final plate.')

    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   30,
                   z_offset=1.5,
                   x_offset=1,
                   rate=0.1,
                   drop_tip=True)

    magblock.disengage()


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo
from scrapy.conf import settings
from scrapy.exceptions import DropItem
from scrapy import log
import csv

'''
class QiushibaikePipeline(object):
    def process_item(self, item, spider):
        return item
'''
class MongoDBPipeline(object):
    def __init__(self):
        connection=pymongo.MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        db=connection[settings['MONGODB_DB']]
        self.collection=db[settings['MONGODB_COLLECTION']]
        out = open('res.csv','a', newline='')
        csv_write = csv.writer(out,dialect='excel')
        header = ['用户名','内容','赞数','评论数']
        csv_write.writerow(header)

    def process_item(self,item,spider):
        valid=True
        for data in item:
            if not data:
                valid=False
                raise DropItem('Missing{0}!'.format(data))
        if valid:
            self.collection.insert(dict(item))
            out = open('res.csv','a', newline='')
            csv_write = csv.writer(out,dialect='excel')
            res = [self.clear_data(str(item['username'])),self.clear_data(str(item['content'])),self.clear_data(str(item['vote'])),self.clear_data(str(item['comments']))]
            csv_write.writerow(res)
            log.msg('question added to mongodb database!',
                    level=log.DEBUG,spider=spider)
        return item

    def clear_data(self,str):
        temp = str
        temp = temp.replace('\\n', "")
        temp = temp.replace("(", "")
        temp = temp.replace(",)", "")
        temp = temp.replace("'", "")

        return temp


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/galaxy/jobs/metrics/instrumenters/meminfo.py
# Compiled at: 2018-09-15 08:40:24
"""The module describes the ``meminfo`` job metrics plugin."""
import re, sys
from galaxy import util
from . import InstrumentPlugin
from .. import formatting
if sys.version_info > (3, ):
    long = int
MEMINFO_LINE = re.compile('(\\w+)\\s*\\:\\s*(\\d+) kB')
MEMINFO_TITLES = {'memtotal': 'Total System Memory',
   'swaptotal': 'Total System Swap'}

class MemInfoFormatter(formatting.JobMetricFormatter):

    def format(self, key, value):
        title = MEMINFO_TITLES.get(key, key)
        return (title, util.nice_size(value * 1000))

class MemInfoPlugin(InstrumentPlugin):
    """ Gather information about processor configuration from /proc/cpuinfo.
    Linux only.
    """
    plugin_type = 'meminfo'
    formatter = MemInfoFormatter()

    def __init__(self, **kwargs):
        self.verbose = util.asbool(kwargs.get('verbose', False))

    def pre_execute_instrument(self, job_directory):
        return "cat /proc/meminfo > '%s'" % self.__instrument_meminfo_path(job_directory)

    def job_properties(self, job_id, job_directory):
        properties = {}
        with open(self.__instrument_meminfo_path(job_directory)) as (f):
            for line in f:
                line = line.strip()
                if not line:
                    continue
                line_match = MEMINFO_LINE.match(line)
                if not line_match:
                    continue
                key = line_match.group(1).lower()
                if key in MEMINFO_TITLES or self.verbose:
                    value = long(line_match.group(2))
                    properties[key] = value

        return properties

    def __instrument_meminfo_path(self, job_directory):
        return self._instrument_file_path(job_directory, 'meminfo')

__all__ = ('MemInfoPlugin', )


import sys
sys.path.append('..')

import core.config as conf
from utils.util import save_memory, split_join

import numpy as np
import pandas as pd, numpy as np, gc
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import pickle

from tqdm import tqdm

def read_data(path, type='csv', sep='\x01', features=conf.raw_features+conf.labels):
    if type == 'csv':
        df = pd.read_csv(f'{path}', sep=sep, header=None, names=features)
        return df
    else:
        print('cannot read data')

def scaling(df, dense_features):
    mms = MinMaxScaler(feature_range = (0, 1))
    df[dense_features] = mms.fit_transform(df[dense_features])
    return df

def grouping(x, base, div, group=None):
    if group :
        x = x-base
        for i in range(len(group)) :
            if x < group[i] :
                return i
        return len(group)
    return (x-base)//div

def label_encoder(df, sparse_features):
    for feat in sparse_features :
        lbe = LabelEncoder()
        df[feat] = lbe.fit_transform(df[feat])
    return df

def feature_extraction(raw_df, features, train=False):
    labels = conf.labels
    if train:
        df = raw_df[features + labels].copy()
        for label in (labels):
            label_name = label.split('_')[0]
            df.loc[df[label]<=0, label_name ] = 0
            df.loc[df[label]>0, label_name ] = 1
            df = df.drop([label], axis=1)
    else:
        df = raw_df[features].copy()
        for label in (labels):
            label_name = label.split('_')[0]
            # df[label_name] = 0

    del raw_df

    # for timestamp
    df['dt_day']  = pd.to_datetime( df['tweet_timestamp'] , unit='s' ).dt.day.values.astype( np.int8 )
    df['dt_dow']  = pd.to_datetime( df['tweet_timestamp'] , unit='s' ).dt.dayofweek.values.astype( np.int8 )
    df['dt_hour'] = pd.to_datetime( df['tweet_timestamp'] , unit='s' ).dt.hour.values.astype( np.int8 )

    save_memory(df)

    if 'media' in (df.columns):
        df['media'] = df['media'].fillna( '' ).copy()
        gc.collect()
        df['media'] = df['media'].apply( lambda x:  '_'.join(x.split('\t')[:2]) )
        gc.collect()

    for col in (['tweet_type','media']): # except language
        if col in df.columns:
            # df[col] = pd.factorize( df[col], sort=True )[0]
            pkl_path = conf.pickle_data+col+'_dict.pkl'
            with open(pkl_path, 'rb') as f:
                dictionary = pickle.load(f)
            df[col] = df[col].apply(lambda x: dictionary[x])

            gc.collect()
            df[col] = df[col].astype( np.uint8 )
            gc.collect()

    # most_frequent_token
    for col in (['domains','links']):
        if col in df.columns:
            df = most_frequent_token(df, col=col, sep='\t')

    return df

def most_frequent_token(df, col, sep='\t'):
    df['len_'+col] = df[col].apply(lambda x: str(x).count('\t')+1 if not(pd.isnull(x)) else 0)
    var = df[col].fillna('').values.copy()
    gc.collect()

    PD = {}
    null_value = ''
    PD[null_value] = [0,0]
    count = 1
    for vs in var:
        if vs != null_value:
            for v in vs.split('\t'):
                if v not in PD:
                    PD[v] = [count,1]
                    count +=1
                else:
                    x = PD[v]
                    x[1] += 1
                    PD[v] = x
        else:
            x = PD[null_value]
            x[1] += 1
            PD[null_value] = x

    vari = []
    for vs in var:
        if vs != null_value:
            li=[]
            lf=[]
            for v in vs.split('\t'):
                if v!='':
                    li.append(PD[v][0])
                    lf.append(-PD[v][1])
            vari.append( list(np.array(li)[np.argsort(lf)].astype(np.int32) ) )
        else:
            vari.append( [0] )
    del PD

    gc.collect()
    df[col] = np.array( [v[0] for v in vari ] ).astype( np.int32 )
    gc.collect()
    del vari
    del var
    gc.collect()

    return df

def tartget_encoding( tra, col, tar, L=1, smooth_method=0):
    np.random.seed(L)

    cols = col+[tar]
    gf = cudf.from_pandas(tra[cols])
    mn = gf[tar].mean().astype('float32')

    predtrain = np.zeros( tra.shape[0] )

    for fold in [7,8,9,10,11,12]:
        px = np.where( tra.dt_day <fold )[0]
        py = np.where( tra.dt_day==fold )[0]
        mn = gf[tar].iloc[px].mean().astype('float32')
        if smooth_method==0:
            te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])
            te['smooth']  = (te['mean']*te['count'])
            te['smooth'] += (mn*L)
            te['smooth'] /= (te['count']+L)
            te = te.drop( ['mean','count'], axis=1 )
        elif smooth_method==1:
            te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])
            te['smooth'] = (te['sum']+L) / (te['count']+1)
            te = te.drop( ['sum','count'], axis=1 )
        gf2 = gf.iloc[py].copy()
        gf2 = gf2.set_index( col )
        gf2['id'] = cupy.arange( gf2.shape[0] )
        gf2 = gf2.join( te, how='left' )
        gf2 = gf2.sort_values( 'id' )
        del te
        del gf2['id']
        predtrain[py] = gf2.smooth.fillna(-999).to_array()
        del gf2

    px = np.where( tra.dt_day <13 )[0]
    py = np.where( tra.dt_day>=13 )[0]
    mn = gf[tar].iloc[px].mean().astype('float32')
    if smooth_method==0:
        te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])
        te['smooth']  = (te['mean']*te['count'])
        te['smooth'] += (mn*L)
        te['smooth'] /= (te['count']+L)
        te = te.drop( ['mean','count'], axis=1 )
    elif smooth_method==1:
        te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])
        te['smooth'] = (te['sum']+L) / (te['count']+1)
        te = te.drop( ['sum','count'], axis=1 )
    gf2 = gf.iloc[py].copy()
    gf2 = gf2.set_index( col )
    gf2['id'] = cupy.arange( gf2.shape[0] )
    gf2 = gf2.join( te, how='left' )
    gf2 = gf2.sort_values( 'id' )
    del te
    del gf2['id']
    predtrain[py] = gf2.smooth.fillna(-999).to_array()
    del gf2

    px = np.where( (tra.dt_day>=7)&(tra.dt_day<=11) )[0]
    py = np.where( tra.dt_day==6 )[0]
    mn = gf[tar].iloc[px].mean().astype('float32')
    if smooth_method==0:
        te = gf.iloc[px].groupby(col)[tar].agg(['mean','count'])
        te['smooth']  = (te['mean']*te['count'])
        te['smooth'] += (mn*L)
        te['smooth'] /= (te['count']+L)
        te = te.drop( ['mean','count'], axis=1 )
    elif smooth_method==1:
        te = gf.iloc[px].groupby(col)[tar].agg(['sum','count'])
        te['smooth'] = (te['sum']+L) / (te['count']+1)
        te = te.drop( ['sum','count'], axis=1 )
    gf2 = gf.iloc[py].copy()
    gf2 = gf2.set_index( col )
    gf2['id'] = cupy.arange( gf2.shape[0] )
    gf2 = gf2.join( te, how='left' )
    gf2 = gf2.sort_values( 'id' )
    del te
    del gf2['id']
    predtrain[py] = gf2.smooth.fillna(-999).to_array()
    del gf2

    _ = gc.collect()
    predtrain[predtrain <= -999 ] = np.nan
    return predtrain.astype(np.float32)


"""
Data generation.
"""

import numpy as np

s_min = -2
s_max = +2

def s_breaks(b):
    """
    :param b: number of breakpoints (must be odd)
    :return: an array spaced between [s_min, s_max], inclusive.

    If b == 1: [0]
    If b == 3: [-2, 0, 2]
    If b == 5: [-2, -1, 0, 1, 2]
    If b == 9: [-2, -1.5, -1, -0.5, 0, 0.5, 1, 1.5, 2]
    """
    return np.linspace(s_min, s_max, num=b)

def s_gaps(g):
    """
    :param g: number of gaps
    :return: a 2 x k array of gaps. Row 0 has start coordinates. Row 1 has
    end coordinates.

    Example return value:

    np.array([
      [0.6555, -0.9614, 1.3497],
      [0.869,  -0.7522, 1.4861]
    ])
    """
    c = np.random.uniform(s_min, s_max, g)
    w = np.abs(np.random.normal(0.15, 0.1, g))
    return np.vstack([c - w, c + w])

def random_s(gaps, max_iter=20):
    """
    :param gaps: gaps in s (from `s_gaps`)
    :param max_iter: maximum iterations
    :return: a random sample of s (scalar double)
    """
    i = 0
    s = None
    while not s and i < max_iter:
        s = random_s_base(gaps)
        i += 1

    if i == max_iter:
        raise(RuntimeError('exceeded max iterations {}'.format(i)))
    else:
        return s

def random_s_base(gaps):
    """
    :param gaps: gaps in s (from `s_gaps`)
    :return: a random sample of s (scalar double) or `None`
    """
    x = np.random.uniform(s_min, s_max, 1)
    for i in range(0, gaps.shape[1]):
        if gaps[0][i] < x < gaps[1][i]:
            return None
    return x[0]

def func_matrix(b, d):
    """
    :param b: number of breakpoints
    :param d: number of observable dimensions
    :return: a `b+1` x `d` array
    """
    a = np.random.normal(0, 0.5, (b + 1) * d)
    return a.reshape((b + 1, d))

def memo_func(breaks, matrix):
    """
    :param breaks:
    :param matrix:
    :return: an array of length `2 * b`
    """
    b = len(breaks)
    a = np.empty((2 * b, matrix.shape[1]))
    i = 0
    for s in breaks:
        a[2 * i] = s * matrix[i]
        a[2 * i + 1] = s * matrix[i + 1]
        i += 1
    return a

def observation(s, breaks, matrix, memo):
    """
    Returns an observation using memoization, based on the parameter s.
    """
    b = len(breaks)
    if b == 0 or s <= breaks[0]:
        return s * matrix[0]
    for i in range(1, b + 1):
        if i == b or s <= breaks[i]:
            j = (i - 1) * 2
            return memo[j] + s * matrix[i] - memo[j + 1]

def noise(d):
    return np.random.normal(0, 0.05, d)

def observations(n, d, b, g):
    """
    :param n: number of observations
    :param d: number of observable dimensions
    :param b: number of breakpoints
    :param g: number of gaps
    :return: a n x d array of observations
    """
    xs = np.empty((n, d), dtype=np.float32)
    breaks = s_breaks(b)
    gaps = s_gaps(g)
    matrix = func_matrix(b, d)
    memo = memo_func(breaks, matrix)
    for i in range(0, n):
        s = random_s(gaps)
        o = observation(s, breaks, matrix, memo)
        o += noise(d)
        xs[i] = o
    return xs


__author__ = 'basilbeirouti'

from DBConnect import concatenated_psums, last_thousand, concatenated_name_psums
from Scheduling import pids
from BM25 import applicationdatafolder
import glob, os, pickle

def current_pids():
    curr_pids = []
    filenames = sorted([filename for filename in glob.glob(os.path.join(os.getcwd(), "TextCorpus") + "/*.txt")])
    for filename in filenames:
        path, name = os.path.split(filename)
        name = name.replace(".txt", "")
        curr_pids.append(name)
    return curr_pids

def fetch_documents():
    for pid in pids:
        document, fn, ln = concatenated_name_psums(pid, 2000)
        if document is None:
            continue
        yield pid, fn, ln, document

def fetch_new_documents():
    curr_docs = current_pids()
    for pid in pids:
        if pid in curr_docs:
            continue
        document, fn, ln = concatenated_name_psums(pid, 2000)
        if document is None:
            continue
        yield pid, fn, ln, document

def corpus_writer(pid_doc_tup):
    pid, fn, ln, document = pid_doc_tup
    with open(os.path.join(os.getcwd(), "TextCorpus", pid + ".txt"), 'w') as file:
        file.write(document)
    return pid, fn, ln

def build_corpus():
    pid_name_tups = list(map(corpus_writer, iter(fetch_documents())))
    return pid_name_tups

def onlynew_build_corpus():
    pid_name_tups = list(map(corpus_writer, iter(fetch_new_documents())))
    return pid_name_tups

def build_pid_dict(pid_name_tups):
    pidsdict = dict()
    for pid, fn, ln in pid_name_tups:
        pidsdict[pid] = (fn, ln)
    return pidsdict

def save_pidsdict(pidsdict, protocol = 2):
    with open(os.path.join(applicationdatafolder(), "pidsdict27.p"), 'wb') as file:
        pickle.dump(pidsdict, file, protocol = protocol)
    return

def load_pidsdict():
    with open(os.path.join(applicationdatafolder(), "pidsdict27.p"), 'rb') as file:
        pidsdict = pickle.load(file)
    return pidsdict

def build_pidsdict():
    pidsdict = dict()
    currentpids = current_pids()
    for pid in currentpids:
        out = last_thousand(pid, 1)
        pidsdict[pid] = (out[0][3], out[0][4])
    return pidsdict

def main():
    pid_name_tups = build_corpus()
    pidsdict = build_pid_dict(pid_name_tups)
    save_pidsdict(pidsdict, 2)
    pidsdict = load_pidsdict()
    return pidsdict

if __name__ == '__main__':
    pidsdict = build_pidsdict()
    save_pidsdict(pidsdict)
    # print(main())


# -*- coding: utf-8 -*-
# Copyright 2019 Green Valley Belgium NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# @@license_version:1.5@@

import cloudstorage
from google.appengine.api.blobstore import blobstore

CHUNK_SIZE = 1024 * 1024

def read_file_in_chunks(file_obj, chunk_size=CHUNK_SIZE):
    while True:
        chunk = file_obj.read(chunk_size)
        if not chunk:
            break
        yield chunk

def is_file_available(filepath):
    try:
        cloudstorage.stat(filepath)
        return True
    except cloudstorage.errors.NotFoundError:
        return False

def upload_to_gcs(file_data, content_type, file_name):
    """
    Args:
        file_data (str or file-like object)
        content_type (unicode)
        file_name (unicode)
    Returns:
        blob_key (unicode): An encrypted `BlobKey` string.
    """
    # this can fail on the devserver for some reason
    with cloudstorage.open(file_name, 'w', content_type=content_type) as f:
        if isinstance(file_data, basestring):
            f.write(file_data)
        else:
            try:
                for chunk in read_file_in_chunks(file_data):
                    f.write(chunk)
            except AttributeError:
                raise ValueError('file_data must be a file-like object')

    return blobstore.create_gs_key('/gs' + file_name).decode('utf-8')


import base64
import logging
import json

import requests
from kafka import KafkaProducer
from kafka.errors import KafkaTimeoutError

from DeviceManager.conf import CONFIG
from DeviceManager.Logger import Log
from datetime import datetime
import time

LOGGER = Log().color_log()

class DeviceEvent:
    CREATE = "create"
    UPDATE = "update"
    REMOVE = "remove"
    CONFIGURE = "configure"
    TEMPLATE = "template.update"

class NotificationMessage:
    event = ""
    data = None
    meta = None

    def __init__(self, ev, d, m):
        self.event = ev
        self.data = d
        self.meta = m

    def to_json(self):
        return {"event": self.event, "data": self.data, "meta": self.meta}

class KafkaNotifier:

    def __init__(self):
        self.kafka_address = CONFIG.kafka_host + ':' + CONFIG.kafka_port
        self.kf_prod = None

        self.kf_prod = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                                bootstrap_servers=self.kafka_address)

        # Maps services to their managed topics
        self.topic_map = {}

    def get_topic(self, service, subject):
        if service in self.topic_map.keys():
            if subject in self.topic_map[service].keys():
                return self.topic_map[service][subject]

        target = "{}/topic/{}".format(CONFIG.data_broker, subject)
        userinfo = {
            "username": "device-manager",
            "service": service
        }

        jwt = "{}.{}.{}".format(base64.b64encode("model".encode()).decode(),
                                base64.b64encode(json.dumps(
                                    userinfo).encode()).decode(),
                                base64.b64encode("signature".encode()).decode())

        response = requests.get(target, headers={"authorization": jwt})
        if 200 <= response.status_code < 300:
            payload = response.json()
            if self.topic_map.get(service, None) is None:
                self.topic_map[service] = {}
            self.topic_map[service][subject] = payload['topic']
            return payload['topic']
        return None

    def send_notification(self, event, device, meta):
        # TODO What if Kafka is not yet up?

        full_msg = NotificationMessage(event, device, meta)
        try:
            topic = self.get_topic(meta['service'], CONFIG.subject)
            LOGGER.debug(f" topic for {CONFIG.subject} is {topic}")
            if topic is None:
                LOGGER.error(f" Failed to retrieve named topic to publish to")

            self.kf_prod.send(topic, full_msg.to_json())
            self.kf_prod.flush()
        except KafkaTimeoutError:
            LOGGER.error(f" Kafka timed out.")

    def send_raw(self, raw_data, tenant):
        try:
            topic = self.get_topic(tenant, CONFIG.subject)
            if topic is None:
                LOGGER.error(f" Failed to retrieve named topic to publish to")
            self.kf_prod.send(topic, raw_data)
            self.kf_prod.flush()
        except KafkaTimeoutError:
            LOGGER.error(f" Kafka timed out.")


import requests

url = 'https://movie.douban.com/top250'
start = range(25, 250, 25)
for i in start:
    params = {
        'start': start,
        'filter': '',
    }
    headers = {
        'User-Agent': ' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'}

    res = requests.get(url=url, params=params, headers=headers)
    print(res.text)


from xai.brain.wordbase.nouns._residency import _RESIDENCY

#calss header
class _RESIDENCIES(_RESIDENCY, ):
	def __init__(self,):
		_RESIDENCY.__init__(self)
		self.name = "RESIDENCIES"
		self.specie = 'nouns'
		self.basic = "residency"
		self.jsondata = {}


# write program that ask user for two numbers.
# the program then calls a function that computes the area of the square with the first number
# and the area of the circle with the second number and states which one is bigger.

def computeArea(num1, num2):
    squareArea = num1 * num1
    circleArea = 3.14 * num2 * num2
    if circleArea> squareArea:
        print('the circle of radius '+str(num2)+' is bigger than the sequare of size ' + str(num1))
    else:
        print('the sequare of size '+str(num1)+' is bigger than the circle of radius ' + str(num2))

first = float(input('Enter the first number: '))
second = float(input('Enter the second number: '))
computeArea(first, second)

input()


import pandas as pd
import requests
import os

class BEA_api:
    """class for calling gross output and quantity index values"""

    def __init__(self, years):
        self.base_url = 'https://apps.bea.gov/api/data/'
        apik  = os.getenv("BEA_API_Key")
        apik = apik.replace("'", "")
        self.base_params = {'UserID': apik, 'method': 'GetData',  # apik is API key
                            'Industry': 'ALL', 'Frequency': 'A',
                            'Format': 'json'}
        self.tableID_dict = {'go_quant_index': 16, 'go_nominal': 15,
                             'va_nominal': 1, 'va_quant_index': 8}
        self.years = years

    def call_api(self, params):
        """
        Method for calling BEA api for given range of years [min, max]

        Parameters
        ----------
        years : list
            List of begnning year and ending year of data to retrieve from
            BEA API.

        params : dict
            Dictionary of DataSetName and tableID.

        Returns
        -------
        data : dataframe
            Dataframe of results. Will print API error, if applicable.
        """

        api_params = self.base_params.copy()
        api_params['Year'] = "All" #\
            #[str(y) for y in range(self.years[0], self.years[1]+1)] #

        for k, v in params.items():
            api_params[k] = v

        r = requests.get(self.base_url, params=api_params)
        print('r:\n', type(r.json()['BEAAPI']['Results']), type(r.json()['BEAAPI']['Results'][0]))

        try:
            data_json = r.json()['BEAAPI']['Results']
            data = [pd.DataFrame.from_records(
                data_json[i]['Data']
                ) for i in range(len(data_json))]

        except KeyError:
            print('API Error:{}'.format(
                r.json()['BEAAPI']['Error']['ErrorDetail']['Description']
                ))

        else:
            return data

    def format_data(self, data):
        """format and pivot called data"""

        data['DataValue'] = data.DataValue.astype(float)

        pivoted_data = data.pivot_table(
            index=['IndustrYDescription', 'Industry'], values='DataValue',
            columns='Year'
            )

        pivoted_data.reset_index(inplace=True)

        return pivoted_data

    def get_data(self, table_name):
        """
        Get data using the BEA API call.

        Parameters
        ----------
        years : list
            List of begnning year and ending year of data to retrieve from
            BEA API.

        table_name : str
            go_quant_index (Industry Gross Output, quantity indexes),
            go_nominal (Industry Gross Output, nominal),
            va_nominal (Industry Value Added, nominal),
            va_quant_index (Industry Value Added, quanitity indexes).
            Additional values can be added with their corresponding tableID
            number to tableID_dict

        Returns
        -------
        data : dataframe
            Dataframe of BEA data.

        """

        api_params = {'DataSetName': 'GDPbyIndustry',
                      'tableID': self.tableID_dict[table_name]}

        data = self.call_api(params=api_params)
        print('data: \n', data)
        print('data: \n', data[0])
        print('data: \n', type(data[0]))

        if data:
            data = [self.format_data(data[i]) for i in range(len(data))]
            return data

        else:
            return

    def import_historical(self):
        """Method for importing historical BEA data, saved as csv
        Footnotes: 1. Consists of agriculture, forestry, fishing, and hunting; mining; construction; and manufacturing.
        2. Consists of utilities; wholesale trade; retail trade; transportation and warehousing; information; finance, insurance, real estate, rental, and leasing; professional and business services; educational services, health care, and social assistance; arts, entertainment, recreation, accommodation, and food services; and other services, except government.
        3. Consists of computer and electronic product manufacturing (excluding navigational, measuring, electromedical, and control instruments manufacturing); software publishers; broadcasting and telecommunications; data processing, hosting and related services; internet publishing and broadcasting and web search portals; and computer systems design and related services.
        """
        historical_va_quant_index = pd.read_csv('./EnergyIntensityIndicators/Industry/Data/Chain_Type_Qty_Indexes_Value_Added_by_Industry.csv').set_index('Industry') # 2012 = 100
        historical_va = pd.read_csv('./EnergyIntensityIndicators/Industry/Data/Historical_VA.csv').set_index('Industry')

        historical_go = pd.read_csv('./EnergyIntensityIndicators/Industry/Data/Historical_GO.csv').set_index('Industry')
        historical_go_quant_index = pd.read_csv('./EnergyIntensityIndicators/Industry/Data/historical_go_qty_index.csv').set_index('Industry') # 2012 = 100

        historical_data = {'historical_va': historical_va, 'historical_va_quant_index': historical_va_quant_index,
                           'historical_go': historical_go, 'historical_go_quant_index': historical_go_quant_index}
        return historical_data

    def chain_qty_indexes(self):
        """Merge historical and api data, manipulate as in ChainQtyIndexes
        """
        historical_data = self.import_historical()
        print('historical_data: \n', historical_data)

        go_nominal = self.get_data(table_name='go_nominal')
        if len(go_nominal) == 1:
            go_nominal = go_nominal[0]
            print('go_nominal: \n', go_nominal)
        else:
            raise TypeError(f'list of go_nominal of len {len(go_nominal)}')

        print('len go_nominal: \n', len(go_nominal))
        historical_go = historical_data['historical_go']
        go_nominal_12 = go_nominal[go_nominal["Year"] == 2012][['DataValue', 'IndustrYDescription', 'Industry']].transpose()
        print('go_nominal_12: \n', go_nominal_12)
        go_quant_index = self.get_data(table_name='go_quant_index')
        min_api_year = min([int(y) for y in go_quant_index.columns])

        historical_go_qty_index = historical_data['historical_go_qty_index']
        historical_go_qty_index = historical_go_qty_index.loc[:, : min_api_year]

        go_quant_index = go_quant_index.merge(historical_go_qty_index, left_index=True, right_index=True, how='outer')
        print('go_quant_index: \n', go_quant_index)

        transformed_go_quant_index = go_quant_index.multiply(go_nominal_12, index=1).multiply(0.01)
        # transformed_go_quant_index.loc['          Transportation equipment', :] =

        transformed_go_quant_index = transformed_go_quant_index.transpose()

        # transformed_go_quant_index is further manipulated but I'm not sure exactly how it works
        transformed_go_quant_index = transformed_go_quant_index.divide(transformed_go_quant_index.loc[self.base_year, :], axis=0)

        va_nominal = self.get_data(table_name='va_nominal')
        print('len va_nominal: \n', len(va_nominal))

        historical_va = historical_data['historical_va']
        va_nominal_12 = va_nominal[va_nominal["Year"] == 2012][['DataValue', 'IndustrYDescription', 'Industry']].transpose()
        print('va_nominal_12:\n', va_nominal_12)
        va_quant_index = self.get_data(table_name='va_quant_index')
        historical_va_qty_index = historical_data['historical_va_qty_index']
        va_quant_index = va_quant_index.merge(historical_va_qty_index, left_index=True, right_index=True, how='outer')

        transformed_va_quant_index = va_quant_index.multiply(va_nominal_12, index=1).multiply(.01)
        transformed_va_quant_index = transformed_va_quant_index.transpose()
        # transformed_va_quant_index is further manipulated but I'm not sure exactly how it works

        go_over_va = transformed_go_quant_index.divide(transformed_va_quant_index)
        return transformed_va_quant_index, transformed_go_quant_index

if __name__ == '__main__':
    data = BEA_api(years=list(range(1949, 2018))).get_data(table_name='go_nominal')
    print(data)


import pickle

from nose.tools import eq_

from .. import indonesian
from ...datasources import diff, parent_revision, revision
from ...dependencies import solve

BAD = [
    "anjing",
    "bajingan",
    "bangsat",
    "bispak",
    "bloon", "blo'on",
    "brengsek", "brengsex",
    "bencong",
    "babi",
    "cibai",
    "coley",
    "diselama",
    "escoduro",
    "fredrike",
    "fogh",
    "gauguin",
    "goblok",
    "gestapo",
    "husseins",
    "indon",
    "jambut",
    "jellinek",
    "jellygamat",
    "keparat",
    "kontol",
    "lonte",
    "loked",
    "lvmh",
    "malingsia",
    "memek",
    "monyong",
    "netnapa",
    "ngentot",
    "nesbitt",
    "panadta",
    "palaji",
    "perek",
    "pukimak",
    "pedofil",
    "riyhad",
    "sempak",
    "sinting",
    "steinway",
    "sukhano",
    "taenjamras",
    "tetek",
    "titit",
    "toket",
    "tzcesar",
    "thailaland",
    "thaicia"
]

INFORMAL = [
    "hai",
    "halo",
    "jancok"
]

OTHER = [
    """
    Gita Gutawa adalah seorang penyanyi sopran, aktris, dan penulis lagu
    berkebangsaan Indonesia. Ia adalah putri dari komponis Erwin Gutawa.
    Meskipun pada awalnya sempat belajar piano, Gita kemudian beralih ke vokal.
    Bakatnya ditemukan pada tahun 2004 saat ia sedang berlatih vokal,
    kemudian diminta untuk berduet dengan ADA Band.
    """
]

def compare_extraction(extractor, examples, counter_examples):

    for example in examples:
        eq_(extractor.process(example), [example])
        eq_(extractor.process("Sentence " + example + " sandwich."), [example])
        eq_(extractor.process("Sentence end " + example + "."), [example])
        eq_(extractor.process(example + " start of sentence."), [example])

    for example in counter_examples:
        eq_(extractor.process(example), [])
        eq_(extractor.process("Sentence " + example + " sandwich."), [])
        eq_(extractor.process("Sentence end " + example + "."), [])
        eq_(extractor.process(example + " start of sentence."), [])

def test_badwords():
    compare_extraction(indonesian.revision.badwords_list, BAD, OTHER)

def test_informals():
    compare_extraction(indonesian.revision.informals_list, INFORMAL, OTHER)

def test_revision():
    # Words
    cache = {revision.text: "Gita Gutawa adalah seorang m80 sopran, aktris."}
    eq_(solve(indonesian.revision.words_list, cache=cache),
        ["Gita", "Gutawa", "adalah", "seorang", "m80", "sopran", "aktris"])

    # Misspellings
    cache = {revision.text: 'Setelah merilis album duet worngly. <td>'}
    eq_(solve(indonesian.revision.misspellings_list, cache=cache), ["worngly"])


import random
import math

lowest = int(input("Enter a number 1 here: "))
highest = int(input("Enter a number 9 here: "))

x = random.randint(lowest, highest)
print("\n\tYou've only have",
       round(math.log(highest - lowest + 1, 2)),
      "chances to guess the number!\n")

count = 0

while count < math.log(highest - lowest + 1, 2):
    count += 1

    guess = int(input("Guess a number: "))

    if x == guess:
        print("Great job, you guessed it on try number: ",
              count)
        break
    elif x > guess:
        print("The number you guessed is too low!")
    elif x < guess:
        print("The number you guessed is too high!")

if count >= math.log(highest - lowest + 1, 2):
    print("\nThe number is %d" % x)
    print("\t Sorry you did not get it correct! Better Luck Next time!")