#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jul 20 15:29:22 2019

@author: sl
"""

import logging
import pandas as pd
import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt

logger = logging.getLogger(__name__)
#logger.setLevel(logging.DEBUG)

class GaussianNB:
    """
    Gaussian Naive Bayes classifier.

    Features are supposed to be real-valued, normally distributed within
    each class.
    """
    def __init__(self):
        self.C = 0  # number of classes
        self.N = 0  # number of features
        self.classes_log_probas = None  # classes prior log probas
        self.classes_labels = None
        self.mus = None  # matrix of Gaussian mean for every class and feature
        self.sigmas = None # matrix of Gausisan std for every class and feature
        self.constants = None # matrix of constants in Gaussian depending only by std
        self.was_fit = False

    def fit(self, X, y):
        # for fit(), X and y are specified in sklearn fashion:
        # X is numpy array of shape (n_samples, n_features)
        # y is numpy array of shape (n_samples)
        dfx = pd.DataFrame(X)
        dfy = pd.Series(y)
        self.classes_log_probas = np.log((dfy.value_counts() / len(y)).sort_index())
        self.classes_labels = self.classes_log_probas.index.values.tolist()
        self.C = len(self.classes_labels)
        self.N = X.shape[1]
        self.mus = np.zeros((self.C, self.N))
        self.sigmas = np.zeros((self.C, self.N))
        self.constants = np.zeros((self.C, self.N))
        # calculate small additive fraction of most wide std from features.
        # it's needed to avoid numerical unstability,
        # when sigma estimate from feature std is too close to 0
        self.std_eps = 1e-4*np.std(X, axis=0).max()
        for i, c in enumerate(self.classes_labels):
            dfxc = dfx[dfy==c]
            means = dfxc.apply(lambda feature:
                np.mean(feature.values, dtype=np.float64), axis=0).values
            stds  = dfxc.apply(lambda feature:
                np.std(feature.values, dtype=np.float64), axis=0).values
            stds += self.std_eps
            constants = -0.5*np.log(2.*np.pi) - np.log(stds)
            self.mus[i] = means
            self.sigmas[i] = stds
            self.constants[i] = constants
        self.was_fit = True
        return self

    def _predict_one(self, x):
        """
        Predict class label given new x, using MAP rule on posterior log probas.
        Posterior log proba logp(c|x) is calculated from Bayes rule:

            logp(y|x) ~~ logp(y) + sum( logp(x|y) )

        for GaussianNB,  p(x|y) ~ N(mu, std)

        Predicted class label is c = argmax logp(y|x), for y in classes_labels
        """
        if not self.was_fit:
            raise ValueError("trying to predict before fit")
        log_pyx = []
        for i, c in enumerate(self.classes_labels):
            log_pc = self.classes_log_probas[c]
            log_pxy = self.constants[i] - (0.5*(x - self.mus[i])**2) / (self.sigmas[i]**2)
            log_pyx.append(log_pc + np.sum(log_pxy))

        return self.classes_labels[np.argmax(np.array(log_pyx))]

    def predict(self, X):
        return np.apply_along_axis(self._predict_one, axis=1, arr=X)

class BernoulliNB:
    """
    Bernoulli Naive Bayes classifier.

    Features are supposed to be boolean.
    If feature values are non-negative real values, they are transformed to boolean.

    Only dense numpy arrays are supported.
    """
    def __init__(self):
        self.C = 0  # number of classes
        self.N = 0  # number of features
        self.classes_log_probas = None  # classes prior log probas
        self.classes_labels = None
        self.logp = None  # matrix of Bernoulli log probas for every class and feature
        self.neglogp = None  # matrix for log(1 - p)
        self.was_fit = False

    def fit(self, X, y):
        # for fit(), X and y are specified in sklearn fashion:
        # X is numpy array of shape (n_samples, n_features)
        # y is numpy array of shape (n_samples)
        assert np.all(X >= 0.)
        dfx = pd.DataFrame(X).astype(bool).astype(int)
        dfy = pd.Series(y)
        self.classes_log_probas = np.log((dfy.value_counts() / len(y)).sort_index())
        self.classes_labels = self.classes_log_probas.index.values.tolist()
        self.C = len(self.classes_labels)
        self.N = X.shape[1]
        self.logp = np.zeros((self.C, self.N))
        self.neglogp = np.zeros((self.C, self.N))
        for i, c in enumerate(self.classes_labels):
            dfxc = dfx[dfy==c]
            p = dfxc.apply(lambda feature:
                (1 + np.count_nonzero(feature.values)) / (2+ len(feature)), axis=0).values
            self.logp[i] = np.log(p)
            self.neglogp[i] = np.log(1. - p)
        self.was_fit = True
        return self

    def _predict_one(self, x):
        """
        Predict class label given new x, using MAP rule on posterior log probas.
        Posterior log proba logp(c|x) is calculated from Bayes rule:

            logp(y|x) ~~ logp(y) + sum( logp(x|y) )

        for BernoulliNB,  logp(x|yi) = xi * log(Pi) + (1 - xi) * log(1 - Pi)

        Predicted class label is c = argmax logp(y|x), for y in classes_labels
        """
        if not self.was_fit:
            raise ValueError("trying to predict before fit")
        log_pyx = []
        for i, c in enumerate(self.classes_labels):
            log_pc = self.classes_log_probas[c]
            log_pxy = x * self.logp[i] + self.neglogp[i] - x * self.neglogp[i]
            log_pyx.append(log_pc + np.sum(log_pxy))

        return self.classes_labels[np.argmax(np.array(log_pyx))]

    def predict(self, X):
        return np.apply_along_axis(self._predict_one, axis=1, arr=X)


from telethon import TelegramClient
from telethon.tl.types import DocumentAttributeVideo
from setting import api_id, api_hash, TOKEN

bot_token = TOKEN
api_id = api_id  # 申请的TG API ID
api_hash = api_hash  # 申请的TG API hash

# file_path = '/tmp/sg.mp4'
# path = '/tmp/qbt.mp4'

async def main(client, file_path):
    # await client.send_file('@dmtgrobot', path, video_note=True, supports_streaming=True, attributes=(DocumentAttributeVideo(1440, 720,720),))
    # await client.send_file("@dmtgrobot", path, supports_streaming=True, attributes=(DocumentAttributeVideo(6*60+42, 1920, 1080),))
    # await client.send_file('@dmtgrobot', path, allow_cache=False,supports_streaming=True, attributes=(DocumentAttributeVideo(1440, 720,720),))
    await client.send_file('@dmtgrobot', file_path, allow_cache=False, supports_streaming=True)

# with client:
def run(file_path):
    client = TelegramClient('anon', api_id, api_hash)
    client.loop.run_until_complete(main(client, file_path))


import os
import subprocess

clip0 = os.environ['clip0']
clip1 = os.environ['clip1']
clip2 = os.environ['clip2']
clip3 = os.environ['clip3']
clip4 = os.environ['clip4']
clip5 = os.environ['clip5']
clip6 = os.environ['clip6']
clip7 = os.environ['clip7']
clip8 = os.environ['clip8']
clip9 = os.environ['clip9']

result = subprocess.run(['osascript', '-e',
                         'if application "Google Chrome" is running then tell application "Google Chrome" to return URL of active tab of front window'
                         ], stdout=subprocess.PIPE)
activeTabUrl = result.stdout.decode('utf-8')

clipboard = filter(None, [
    activeTabUrl,
    clip0,
    clip1,
    clip2,
    clip3,
    clip4,
    clip5,
    clip6,
    clip7,
    clip8,
    clip9,
])


from settings import *
import pygame
import random
import pyperclip
import threading
import os
import time
from math import cos, sin

pygame.init()
os.environ['SDL_VIDEO_CENTERED'] = '1' #center window
clock = pygame.time.Clock()
win = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption(TITLE)

colors = [pygame.Color("red"), pygame.Color("purple"), pygame.Color("pink"),
		pygame.Color("white"), pygame.Color("yellow"), pygame.Color("lightblue")]

abc = "abcdefghijklmnopqrstuvwxyz"

prvi=["Ku","Ste","Ser","Gri","Di","Ni","Pa","Bo","A","Je","Ta","Gav","Mi","Sla","Sne","Snje","Di","Du","Da","Bor","Bran","Zdrav","Ra","Lje","Vu","Bo","Voj","Ja","Ran","Nov","Su","Sve","Svje",
		  "Zvje","Zve","Sve","Dmi","Slav","Mir","Rat","Rad","Boj","Stra","Ne","Ve","Sun","Ma","Ta","Fej","Ju","Je","I","Vla"]
sred=["go","ri","mi","tri","ko","li","sta","si","li","sa","ve","tja","lo","ri","dja","za","ja","brav","bor","doj","po","sa","go","sla","dran","za","tla","zda","mir","ni","lja","boj","hi",
		  "ci","ri","ja","hu","a","dzu","tu","re","mi","ho","di","mir","sti"]
zadn=["fan","gej","je","na","vle","jo","ja","ta","lo","mir","ca","djan","ven","ka","va","ljub","slav","ko","tar","sa","nja","bor","ja","din","lah","suf","da","sus","dolf","sta","sto"]

def namer():
		middle=''
		for _ in range(random.randrange(0,3)):
				middle=''.join([middle, random.choice(sred)])
		first = random.choice(prvi)
		last = random.choice(zadn)
		return first+middle+last

def planet_namer():
	output = ""
	for _ in range(random.randint(3,15)):
		output += abc[random.randint(0, 25)]

	return output

class StarType:
	def __init__(self, name, color, temperature, life=False, chance=(0,40)):
		self.name = name #Ime vrste // string
		self.color = color #Boja zvijezde // tuple(R,G,B)
		self.temperature = temperature #Temperatura planeta u celsiusu // tuple(od, do)
		self.life = life #Dal planete mogu imati zivot // True False
		self.chance = chance

#ovdje dodavaj vrste zvijezda
star_types = [
	StarType("Gentle", (255,200,255), (10,40), life=True),
	StarType("Temperate", (255,81,12), (150, 450)), #pakao
	StarType("Frozen", (125,192,255), (-300, -100)),
	StarType("Radioactive", (255,234,100), (30,70), life=True),
	StarType("Acid", (160,255,0), (-120, 120), chance=(0,80)),
	StarType("Black Hole", (10,10,10), (-273, -200), chance=(0,250)),
	StarType("White Hole", (255,255,255), (69, 420), chance=(0,1000), life=True)
]

class Planet:
	def __init__(self):
		self.radius = 0
		self.name = ""
		self.t = 0
		self.reversedRotation = False
		self.hasWater = False
		self.life = False
		self.ring = False
		self.temperature = 0 #celsius
		self.gas_giant = False
		self.moons = []

		self.gases = 0.0
		self.water = 0.0
		self.minerals = 0.0
		self.resources = 0.0
		self.population = 0

class Star:
	def __init__(self, x, y, generateSystem=True):
		seed = (x & 0xFFFF) << 16 | (y & 0xFFFF)
		random.seed(seed)
		self.type = random.choice(star_types)
		self.starExists = random.randint(*self.type.chance) == 1
		if not self.starExists:
				return

		self.color = self.type.color
		self.radius = random.randint(5, 22)

		#Ako mi treba samo da prikaze zvijezdu ne mora onda generisati sve planete i detalje o njoj
		#nego samo izgled
		if not generateSystem:
			return

		self.name = namer()
		self.planets = []
		n_planets = random.randint(0, 6)
		for _ in range(n_planets):
			p = Planet()
			p.name = namer()
			p.radius = random.randint(5,12)
			p.reversedRotation = random.randint(0,150)==1
			p.t = random.randint(0,360)
			nMoons = random.randint(-5, 3) #-5 da budu vece sanse da nemaju nista nego da svaka ima satelit
			for _ in range(nMoons):
				t = random.randint(0,360)
				reversedRotation = random.randint(0,20)==1
				p.moons.append([t, reversedRotation])

			p.ring = random.randint(0,10) == 1
			p.temperature = random.randint(*self.type.temperature)

			if p.radius >= 10:
				p.gas_giant = random.randint(0,5) == 1

			if not p.gas_giant:
				if p.temperature > 0 and p.temperature < 100:
					p.hasWater = random.randint(0,50) == 1
					if p.hasWater and self.type.life:
						p.life = random.randint(0,10) == 1

			if p.gas_giant:
				p.gases = 1.0
				p.minerals = 0.0
				p.resources = 0.0
				p.water = 0.0
			else:
				p.gases = random.random()
				p.minerals = random.random()
				p.resources = random.random()
				if p.hasWater: p.water = random.random()
			final = 1.0 / (p.gases + p.minerals + p.resources + p.water)
			p.gases *= final * 100
			p.minerals *= final * 100
			p.resources *= final * 100
			p.water *= final * 100
			#Ovaj dio sluzi da izracuna procenat %
			self.planets.append(p)

class Camera:
	def __init__(self):
		self.x = 0
		self.y = 0
		self.speed = 1

	def move(self):
		pressed = False
		key = pygame.key.get_pressed()
		speed = self.speed
		if key[pygame.K_LSHIFT]:
			speed += 2
			if key[pygame.K_LCTRL]:
				speed += 100

		if key[pygame.K_w]:
				self.y -= speed
				pressed = True

		if key[pygame.K_s]:
				self.y += speed
				pressed = True

		if key[pygame.K_a]:
				self.x -= speed
				pressed = True

		if key[pygame.K_d]:
				self.x += speed
				pressed = True

		if key[pygame.K_v]:
			#ako se pritisne V onda uzme od clipboarda podatke
			try:
				x, y = pyperclip.paste().split(":")
				x, y = int(x), int(y)
				self.teleport(x, y)
			except:
				return

		return pressed

	def teleport(self, x, y):
		self.x = x
		self.y = y

cam = Camera()

font = pygame.font.SysFont("Arial", 30)
font_star = pygame.font.SysFont("Arial", 12, bold=True)
font_info = pygame.font.SysFont("Arial", 25)

def get_mouse_in_segment(galaxy=True):
	mos_pos = pygame.mouse.get_pos()
	mos_pos = (mos_pos[0]//SEGMENTS, mos_pos[1]//SEGMENTS) #pozicija misa na ekranu
	mos_galaxy = (mos_pos[0] + cam.x, mos_pos[1] + cam.y) #pozicija misa u svemiru

	if galaxy:
		return mos_galaxy
	else:
		return mos_pos

def clicked():
	click = pygame.mouse.get_pressed()

	if click[0]:
		return 1
	elif click[2]:
		return 2

def command():
	#Konzola thread
	print("To change coords use this format - x:y or use WASD")
	while True:
		coords = input(">>> ")
		try:
			x, y = coords.split(":")
			x, y = int(x), int(y)
		except:
			continue

		cam.teleport(x, y)

threading.Thread(target=command, daemon=True).start()

width = HALF_WIDTH
height = HEIGHT
border = 1

selected = False
selectedStar = [0,0] #x,y

running = True
while running:
	clock.tick(60)
	win.fill(BLACK)

	cam.move()

	for x in range(SECTORS_X):
		for y in range(SECTORS_Y):
			#pygame.draw.rect(win, WHITE, (x*SEGMENTS,y*SEGMENTS,SEGMENTS,SEGMENTS), 1)
			star = Star(x + cam.x, y + cam.y, False)
			if star.starExists:
				center = (x*SEGMENTS+SEGMENTS//2, y*SEGMENTS+SEGMENTS//2)
				pygame.draw.circle(win, star.color, center, star.radius)
				#renderuje zvijezde bez generisanja detalja (samo izgleda)

	if selected:
		x, y = selectedStar
	else:
		x, y = get_mouse_in_segment()
	star = Star(x, y)
	if star.starExists:
		x, y = x-cam.x, y-cam.y
		center = (x*SEGMENTS+SEGMENTS//2, y*SEGMENTS+SEGMENTS//2)
		if x > -SECTORS_X and x < SEGMENTS:
			if y > -SECTORS_Y and y < SEGMENTS:
				#ako je x,y u ekranu onda radi slijedece:
				pygame.draw.circle(win, WHITE, center, star.radius+5, 1)
				name_text = font_star.render(star.name, True, WHITE)
				name_w = name_text.get_width()
				name_h = name_text.get_height()
				win.blit(name_text, (center[0] - name_w//2,center[1] + star.radius+5))
				try:
					pos_text = font_star.render(f"x:{x+cam.x} y: {y+cam.y}", True, WHITE)
					pos_w = pos_text.get_width()
					win.blit(pos_text, (center[0] - pos_w//2,center[1] +star.radius+20))
				except:
					pass

		rel_x = 0
		rel_y = HEIGHT-height - border

		if x*SEGMENTS <= rel_x + width:
			#if y*SEGMENTS >= rel_y and y*SEGMENTS <= rel_y+height:
			#ako je x na lijevoj strani onda prikaze podatke na desnoj
			rel_x = WIDTH-width-border
			rel_y = 0 - border

		rel_x_center = rel_x+width//2
		rel_y_center = rel_y+height//2
		left_bottom = (rel_x, height)

		pygame.draw.rect(win, BLACK, (rel_x, rel_y, width, height))
		pygame.draw.rect(win, WHITE, (rel_x, rel_y, width, height), 2)

		pygame.draw.circle(win, star.color, (rel_x_center, rel_y_center), star.radius)
		mos_x, mos_y = pygame.mouse.get_pos()

		texts = []
		texts.append(font_info.render(f"Name: {star.name}", True, WHITE))
		texts.append(font_info.render(f"Type: {star.type.name}", True, WHITE))
		texts.append(font_info.render(f"Number of planets: {len(star.planets)}", True, WHITE))

		text_width = texts[2].get_width()
		text_height = texts[0].get_height()
		text_offset = -text_height
		for text in reversed(texts):
			win.blit(text, (rel_x+width-text_width, height+text_offset))
			text_offset -= text_height
			#Render textova

		orbit = star.radius*2 + 25
		for n, planet in enumerate(star.planets):
			n += 1
			planet.t += -time.time()/n if planet.reversedRotation else time.time()/(n**2)

			planet_x = orbit * cos(planet.t) + rel_x_center
			planet_y = orbit * sin(planet.t) + rel_y_center

			pygame.draw.circle(win, WHITE, (rel_x_center, rel_y_center), orbit, 1)

			if planet.moons:
				moon_orbit = planet.radius + 10
				for n, (t, reversedRotation) in enumerate(planet.moons):
					n += 1 #da nebi bilo djeljenja sa 0
					t += -time.time()/n if reversedRotation else time.time()/n
					moon_x = moon_orbit * cos(t) + planet_x
					moon_y = moon_orbit * sin(t) + planet_y
					moon_size = random.randint(3,5)
					pygame.draw.circle(win, WHITE, (int(planet_x), int(planet_y)), moon_orbit, 1)
					pygame.draw.circle(win, WHITE, (int(moon_x), int(moon_y)), moon_size)
					moon_orbit += 10

			if selected:
				if (mos_x - rel_x_center)**2 + (mos_y - rel_y_center)**2 < (orbit+10)**2:
					if (mos_x - rel_x_center)**2 + (mos_y - rel_y_center)**2 > (orbit-10)**2:
						pygame.draw.circle(win, WHITE, (int(planet_x), int(planet_y)), planet.radius+3, 2)
						pygame.draw.circle(win, WHITE, (rel_x_center, rel_y_center), orbit+1, 3)

						texts = []

						texts.append(font_info.render(f"Gases: {round(planet.gases)}%", True, WHITE))
						texts.append(font_info.render(f"Minerals: {round(planet.minerals)}%", True, WHITE))
						texts.append(font_info.render(f"Resources: {round(planet.resources)}%", True, WHITE))
						texts.append(font_info.render(f"Water: {round(planet.water)}%", True, WHITE))
						texts.append(font_info.render(f"Temperature: {planet.temperature}C", True, WHITE))
						texts.append(font_info.render(f"Name: {planet.name}", True, WHITE))

						text_height = texts[0].get_height()
						text_offset = -text_height
						for text in texts:
							win.blit(text, (rel_x+3, height+text_offset))
							text_offset -= text_height

			pygame.draw.circle(win, BLUE if planet.water and not planet.life else GREEN if planet.life else GAS if planet.gas_giant else BROWN,
							(int(planet_x), int(planet_y)), planet.radius)
			orbit += 45

		if clicked() == 1:
			if mos_x >= rel_x and mos_x <= rel_x+width:
				pass
			else:
				selected = False

	if clicked() == 1:
		x, y = get_mouse_in_segment()
		star = Star(x, y, False)
		if star.starExists:
			pyperclip.copy(f"{x - (SECTORS_X//2)}:{y - (SECTORS_Y//2)}")
			if not selected:
				selected = True
				selectedStar = [x,y]

	try:
			cords = font.render(f"X:{str(cam.x)}, Y: {str(cam.y)}", True, WHITE)
	except:
			cords = font.render(f"You are too far from the center. Don't get lost", True, WHITE)
	fps = font.render(f"FPS: {str(int(clock.get_fps()))}", True, WHITE)
	win.blit(fps, FPS_POS)
	win.blit(cords, CORD_POS)

	pygame.display.flip()
	for event in pygame.event.get():
			if event.type == pygame.QUIT:
					running = False

pygame.quit()
quit()


#!/usr/bin/env python
"""
Script to calculate the number of salt bridges in a given PDB file of a protein.

Usage: calculate_salt_bridge.py pdbfile [csvfile]

Author: Bingrui Yao <ybrprivate@outlook.com>

Mechanism：
    According to https://sbl.inria.fr/doc/Pointwise_interactions-user-manual.html
    To describe salt bridges, recall that atoms of interest for acidic residues are:
    · D (ASP): OD2
    · E (GLU): OE2
    Likewise, for basic residues:
    · R (ARG): NH1 and NH2
    · K (LYS): NZ
    · H (HIS): ND1 and NE2
    For two of them, if the minimum distance found is less than 3.4, and more than 2.5，the pair is termed a salt bridge.
"""

import argparse
import pandas as pd
import numpy as np
from tqdm import tqdm
import os

# Output directory for salt bridge number files
output_directory = r'/home/**/salt_bridge_number'
# Input directory for PDB files
input_directory = r'/home/**/input_pdbfiles'

def count_salt_bridges(input_file, output_file):
    """
    Count the number of salt bridges in a PDB file.

    Parameters:
    input_file (str): Path to the input PDB file.
    output_file (str): Path to the output CSV file.

    Returns:
    result_df (pd.DataFrame): DataFrame containing the salt bridge count for each PDB file.
    """

    result_dic = {}
    result_bonds = ''

    # Read the PDB file
    pdb_df = pd.read_csv(input_file, delimiter='\s+', header=None)

    # Data processing to extract relevant information
    pdb_df = pdb_df.drop(pdb_df.columns[[0, 1, 4, 5, 9, 10, 11]], axis=1)
    pdb_df.columns = ['Atom_Name', 'Amino_Acid', 'X', 'Y', 'Z']

    # Filter acidic residues in df1，basic residues in df2：
    df1 = pdb_df[((pdb_df['Amino_Acid'] == 'ASP') & (pdb_df['Atom_Name'] == 'OD2')) |
                 ((pdb_df['Amino_Acid'] == 'GLU') & (pdb_df['Atom_Name'] == 'OE2'))]
    df2_1 = pdb_df[(pdb_df['Amino_Acid'] == 'ARG') & (pdb_df['Atom_Name'].isin(['NH1', 'NH2']))]
    df2_2 = pdb_df[(pdb_df['Amino_Acid'] == 'LYS') & (pdb_df['Atom_Name'] == 'NZ')]
    df2_3 = pdb_df[(pdb_df['Amino_Acid'] == 'HIS') & (pdb_df['Atom_Name'].isin(['ND1', 'NE2']))]
    df2 = pd.concat([df2_1, df2_2, df2_3], axis=0)

    # Extract coordinates
    df1_coordinate = df1.loc[:, ['X', 'Y', 'Z']]
    df2_coordinate = df2.loc[:, ['X', 'Y', 'Z']]
    M, N = df1.shape[0], df2.shape[0]
    dist_matrix = np.zeros((M, N))

    # Compute the pairwise Euclidean distances
    for i in range(M):
        for j in range(N):
            dist_matrix[i, j] = np.linalg.norm(df1_coordinate.iloc[i] - df2_coordinate.iloc[j])

    # Filter distances based on criteria
    condition = (dist_matrix > 2.5) & (dist_matrix < 3.4)
    salt_bridge_number = np.count_nonzero(condition)

    # Get PDB ID from input file path
    pdb_id = os.path.splitext(os.path.basename(input_file))[0]

    result_dic[pdb_id] = salt_bridge_number
    result_df = pd.DataFrame.from_dict(result_dic, orient='index', columns=['Salt_Bridge_Number'])
    result_df.to_csv(output_directory + '/' + output_file.split('.')[0] + '.csv', index=True)
    return result_df

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Calculate salt bridges from input files')
    parser.add_argument('-inputfile', type=str, help='Input PDB file name，e.g example_input.pdb')
    parser.add_argument('-outputfile', type=str,
                        help='Output csv file name which save the results，e.g example_output.csv')
    args = parser.parse_args()
    input_file = args.inputfile
    output_file = args.outputfile

    count_salt_bridges(input_file, output_file)


import pathlib
import matplotlib.pyplot as plt

def get_intermediate_results(file_name, knots, verbose):
    with open(file_name) as f:
        motions = []
        for index, line in enumerate(f):
            motion = line.strip().split(' ')
            motions.append(motion)
    K = knots
    x, y = [0] * K, [0] * K
    tail_positions = []
    for motion in motions:
        direction = motion[0]
        steps = int(motion[1])
        if verbose:
            print('\n{} {} '.format(direction, steps), end='')
        for step in range(steps):
            for k in range(K):
                if k == 0:
                    if direction == 'R':
                        x[k] += 1
                    elif direction == 'L':
                        x[k] -= 1
                    elif direction == 'U':
                        y[k] += 1
                    elif direction == 'D':
                        y[k] -= 1
                    if verbose:
                        print('\n{}={},{} '.format('H', x[k], y[k]), end='')
                    continue
                dx = x[k-1] - x[k]
                dy = y[k-1] - y[k]
                if abs(dx) == 2 and abs(dy) == 0:
                    x[k] += int(dx/2)
                elif abs(dx) == 0 and abs(dy) == 2:
                    y[k] += int(dy/2)
                elif abs(dx) == 2 and abs(dy) == 2:
                    x[k] += int(dx/2)
                    y[k] += int(dy/2)
                elif abs(dx) == 2 and abs(dy) == 1:
                    x[k] += int(dx/2)
                    y[k] += dy
                elif abs(dx) == 1 and abs(dy) == 2:
                    x[k] += dx
                    y[k] += int(dy/2)
                if verbose:
                    print('{}={},{} '.format(k, x[k], y[k]), end='')
                if k == K-1:
                    tail_positions.append((x[k],y[k]))
    return tail_positions

def puzzle(file_name, knots, verbose = False, plot = False):
    tail_positions = get_intermediate_results(file_name, knots, verbose)
    unique_tail_positions = dict()
    for tail_position in tail_positions:
        unique_tail_positions[tail_position] = unique_tail_positions.get(tail_position, 0) + 1
    result = len(unique_tail_positions)
    if plot:
        transpose_unique_tail_positions = list(zip(*unique_tail_positions.keys()))
        x = transpose_unique_tail_positions[0]
        y = transpose_unique_tail_positions[1]
        fig, ax = plt.subplots()
        ax.plot(x, y, 'ro')
        plt.grid()
        plt.show(block=True)
    if verbose:
        print('{} --> {} --> {}'.format(file_name, unique_tail_positions, result))
    return result

__location__ = pathlib.Path(__file__).parent

def test_puzzle():
    assert(puzzle(__location__ / 'day9_test.txt', 2, verbose=False) == 13)

def test_puzzle_part2():
    assert(puzzle(__location__ / 'day9_test.txt', 10, verbose=False) == 1)
    assert(puzzle(__location__ / 'day9_test_part2.txt', 10, verbose=False) == 36)

def print_puzzle():
    print(puzzle(__location__ / 'day9_puzzle.txt', 2))

def print_puzzle_part2():
    print(puzzle(__location__ / 'day9_puzzle.txt', 10))

if __name__ == '__main__':
    test_puzzle()
    print_puzzle()
    test_puzzle_part2()
    print_puzzle_part2()


"""
Options file that can be imported and used for unfolding.  Good to separate
data from behavior.
"""
import os
import ROOT as rt
import ordereddict
from tianlu import directories
from tianlu.analysis.utils import SELECTIONPAIR
from tianlu.analysis.P0DCutUtils import getLayerCutStr
from collections import defaultdict

__author__ = 'Tianlu Yuan'
__email__ = 'tianlu.yuan [at] colorado.edu'

#################### Globals ######################
# Set debug to false when analysis is finalized and data can be used
DEBUG = False
# Use T2K style for DrawingTools
T2KSTYLE = False
# production version to use as inputs
PROD_MCP = 'prod6B'
PROD_RDP = 'prod6E'
# highland analysis specifier. need to change SIGNAL and CUTS if this changes
ANALYSIS = 'RunP0DNumuCCQEAnalysis'
# Set to True to use merged MC files, otherwise will use a single
# output instead of merged. If true, a cache file is saved to speed up
# later processing if xsToolsUnfold.stack_sources is called
FULLSTATS = True
# flag for whether to set the fake-data to data POT
FAKEPOT = True
#################### Inputs ######################
FILE_SPEC = 'merged' if FULLSTATS else '1'

# MCP NEUT
NEUT_TEMPLATE = os.path.join(directories.HIGHLAND2_STORE, ANALYSIS, PROD_MCP,
                             'mcp', 'neut', '{0}',
                             '{0}_{1}_{2}'.format(ANALYSIS,
                                                  PROD_MCP,
                                                  'mcp_neut_{0}_'+FILE_SPEC+'.root'))
MCP_NEUT_RUN1 = NEUT_TEMPLATE.format('RUN1')
MCP_NEUT_RUN2A = NEUT_TEMPLATE.format('RUN2air')
MCP_NEUT_RUN2W = NEUT_TEMPLATE.format('RUN2water')
MCP_NEUT_RUN3C = NEUT_TEMPLATE.format('RUN3')
MCP_NEUT_RUN4W = NEUT_TEMPLATE.format('RUN4water')
MCP_NEUT_RUN4A = NEUT_TEMPLATE.format('RUN4air')

# MCP GENIE
GENIE_TEMPLATE = os.path.join(directories.HIGHLAND2_STORE, ANALYSIS, PROD_MCP,
                              'mcp', 'genie', '{0}',
                              '{0}_{1}_{2}'.format(ANALYSIS,
                                                   PROD_MCP,
                                                   'mcp_genie_{0}_'+FILE_SPEC+'.root'))
MCP_GENIE_RUN1 = GENIE_TEMPLATE.format('RUN1')
MCP_GENIE_RUN2A = GENIE_TEMPLATE.format('RUN2air')
MCP_GENIE_RUN2W = GENIE_TEMPLATE.format('RUN2water')
MCP_GENIE_RUN3C = GENIE_TEMPLATE.format('RUN3')
MCP_GENIE_RUN4W = GENIE_TEMPLATE.format('RUN4water')
MCP_GENIE_RUN4A = GENIE_TEMPLATE.format('RUN4air')

# RDP
BEAM_TEMPLATE = os.path.join(directories.HIGHLAND2_STORE, ANALYSIS, PROD_RDP,
                             'rdp', 'beam', '{0}',
                             '{0}_{1}_{2}'.format(ANALYSIS,
                                                  PROD_RDP,
                                                  'rdp_beam_{0}_'+FILE_SPEC+'.root'))
RDP_RUN1 = BEAM_TEMPLATE.format('RUN1')
RDP_RUN2A = BEAM_TEMPLATE.format('RUN2air')
RDP_RUN2W = BEAM_TEMPLATE.format('RUN2water')
RDP_RUN3C = BEAM_TEMPLATE.format('RUN3c')
RDP_RUN4W = BEAM_TEMPLATE.format('RUN4water')
RDP_RUN4A = BEAM_TEMPLATE.format('RUN4air')

# Weights
# t2krw
WEIGHTS_TEMPLATE = os.path.join(ANALYSIS, PROD_MCP, 'mcp', 'neut',
                                '{0}', FILE_SPEC, 'weights_unified.root')
WEIGHTS_NEUT_RUN1 = WEIGHTS_TEMPLATE.format('RUN1')
WEIGHTS_NEUT_RUN2W = WEIGHTS_TEMPLATE.format('RUN2water')
WEIGHTS_NEUT_RUN2A = WEIGHTS_TEMPLATE.format('RUN2air')
WEIGHTS_NEUT_RUN3C = WEIGHTS_TEMPLATE.format('RUN3')
WEIGHTS_NEUT_RUN4W = WEIGHTS_TEMPLATE.format('RUN4water')
WEIGHTS_NEUT_RUN4A = WEIGHTS_TEMPLATE.format('RUN4air')
# mass
MASS_TEMPLATE = os.path.join(ANALYSIS, PROD_MCP, 'mcp', 'neut',
                             '{0}', FILE_SPEC, 'weights_mass.root')
MASS_NEUT_RUN1 = MASS_TEMPLATE.format('RUN1')
MASS_NEUT_RUN2W = MASS_TEMPLATE.format('RUN2water')
MASS_NEUT_RUN2A = MASS_TEMPLATE.format('RUN2air')
MASS_NEUT_RUN3C = MASS_TEMPLATE.format('RUN3')
MASS_NEUT_RUN4W = MASS_TEMPLATE.format('RUN4water')
MASS_NEUT_RUN4A = MASS_TEMPLATE.format('RUN4air')

# xsTool generated rwfiles, needed to get NRooTrackerVtxs
RWFILE_DEFAULT_TEMPLATE = os.path.join(ANALYSIS, PROD_MCP, 'mcp', 'neut',
                                       '{0}', FILE_SPEC, 'rwfile_default.root')
RWFILE_DEFAULT_NEUT_RUN1 = RWFILE_DEFAULT_TEMPLATE.format('RUN1')
RWFILE_DEFAULT_NEUT_RUN2W = RWFILE_DEFAULT_TEMPLATE.format('RUN2water')
RWFILE_DEFAULT_NEUT_RUN2A = RWFILE_DEFAULT_TEMPLATE.format('RUN2air')
RWFILE_DEFAULT_NEUT_RUN3C = RWFILE_DEFAULT_TEMPLATE.format('RUN3')
RWFILE_DEFAULT_NEUT_RUN4W = RWFILE_DEFAULT_TEMPLATE.format('RUN4water')
RWFILE_DEFAULT_NEUT_RUN4A = RWFILE_DEFAULT_TEMPLATE.format('RUN4air')

RWFILE_TRUTH_TEMPLATE = os.path.join(ANALYSIS, PROD_MCP, 'mcp', 'neut',
                                     '{0}', FILE_SPEC, 'rwfile_truth.root')
RWFILE_TRUTH_NEUT_RUN1 = RWFILE_TRUTH_TEMPLATE.format('RUN1')
RWFILE_TRUTH_NEUT_RUN2W = RWFILE_TRUTH_TEMPLATE.format('RUN2water')
RWFILE_TRUTH_NEUT_RUN2A = RWFILE_TRUTH_TEMPLATE.format('RUN2air')
RWFILE_TRUTH_NEUT_RUN3C = RWFILE_TRUTH_TEMPLATE.format('RUN3')
RWFILE_TRUTH_NEUT_RUN4W = RWFILE_TRUTH_TEMPLATE.format('RUN4water')
RWFILE_TRUTH_NEUT_RUN4A = RWFILE_TRUTH_TEMPLATE.format('RUN4air')
##################### End #######################

##################### Flux ######################
FLUX_RUN1 = '{0}/beam/flux/13a/tuned13av1.1/run1/nd5_tuned13av1.1_13anom_run1_fine.root'.format(directories.NFS_BASE)
FLUX_RUN2 = '{0}/beam/flux/13a/tuned13av1.1/run2/nd5_tuned13av1.1_13anom_run2_fine.root'.format(directories.NFS_BASE)
FLUX_RUN3C = '{0}/beam/flux/13a/tuned13av1.1/run3c/nd5_tuned13av1.1_13anom_run3c_fine.root'.format(directories.NFS_BASE)
FLUX_RUN4 = '{0}/beam/flux/13a/tuned13av1.1/run4/nd5_tuned13av1.1_13anom_run4_fine.root'.format(directories.NFS_BASE)
FLUX_HIST_NUMU = 'enu_nd5_tuned13a_numu'
# for xsReweightSystematics.  This specifies which neutrino is used for the
# flux weights
NEUTRINO = 'numu'

################ fake-data POT ##################
# for fake-data studies
# taken from the actual highland2 merged data outputs and retrieved
# using xsInputdataHighland2::GetPOTdata
if FAKEPOT:
     FAKEPOT_RUN1 = 1.67e19
     FAKEPOT_RUN2W = 4.29e19
     FAKEPOT_RUN2A = 3.55e19
     FAKEPOT_RUN3C = 1.35e20
     FAKEPOT_RUN4W = 1.63e20
     FAKEPOT_RUN4A = 1.76e20
else:
    FAKEPOT_RUN1 = -1
    FAKEPOT_RUN2W = -1
    FAKEPOT_RUN2A = -1
    FAKEPOT_RUN3C = -1
    FAKEPOT_RUN4W = -1
    FAKEPOT_RUN4A = -1
##################### End #######################

################## Selection ####################
# (T.Y. 5/6/14) && particle ==13'
SELECT = SELECTIONPAIR('Sum$(accum_level[0][0]) > 3', 'topology==0')
INCLUSIVE = SELECTIONPAIR('Sum$(accum_level[0][0]) > 2', 'topology<3')
SELECT_2TRACK0MICHEL = SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && nmichel==0 && np0d==2', 'topology==0')
##################### End #######################

################### Sidebands ####################
# list all sidebands to be added here
SIDEBANDS = [SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d == 2 && nmichel>0', 'topology == 1'),
             SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d > 2', 'topology == 2')]
# disable for specific sources
SIDEBAND_DISABLED = ['T2KReWeight_flux', 'Mass_all']
##################### End #######################

################# Backgrounds ###################
# Only the background components.
SELECT_BACKGROUND = SELECTIONPAIR('Sum$(accum_level[0][0]) > 3 && topology!=0',
                                  'topology!=0')
SELECT_CC1PI = SELECTIONPAIR('Sum$(accum_level[0][0]) > 3 && topology==1',
                             'topology==1')
SELECT_CCOTHER = SELECTIONPAIR('Sum$(accum_level[0][0]) > 3 && topology==2',
                               'topology==2')
SIDEBAND1_CC1PI = SELECTIONPAIR('&&'.join(SIDEBANDS[0]),
                                SIDEBANDS[0].true)
SIDEBAND2_CCOTHER = SELECTIONPAIR('&&'.join(SIDEBANDS[1]),
                                  SIDEBANDS[1].true)
##################### End #######################

################ Inefficiency ###################
MISSED = SELECTIONPAIR('Sum$(accum_level[0][0]) <=3 && topology==0', 'topology==0')
##################### End #######################

########### Categories Sideband Fit #############
# for category fitting
CATEGS = ['topology==1', 'topology==2']
##################### End #######################

##################### Errors ####################
COLORS = defaultdict(lambda:rt.kOrange,
                     Mass_all=rt.kOrange+2,
                     statistics_data=rt.kCyan,
                     statistics_mc=rt.kCyan+2,
                     T2KReWeight_fsi=rt.kRed,
                     T2KReWeight_xs=rt.kYellow,
                     T2KReWeight_flux=rt.kGreen)
# List here the groups of error sources that we want to include.
# Sort in order of decreasing number of sources for drawing on top of previous
# Empty list corresponds to all implemented errors.
MASTER = ordereddict.OrderedDict()
MASTER['detector'] = (['statistics_data',
                       'statistics_mc',
                       'T2KReWeight_fsi',
                       'T2KReWeight_xs',
                       'T2KReWeight_flux',
                       'Mass_all',
                       'highland_all_syst'], COLORS['detector'])
MASTER['mass'] = (['statistics_data',
                   'statistics_mc',
                   'T2KReWeight_fsi',
                   'T2KReWeight_xs',
                   'T2KReWeight_flux',
                   'Mass_all'], COLORS['Mass_all'])
MASTER['flux'] = (['statistics_data',
                   'statistics_mc',
                   'T2KReWeight_fsi',
                   'T2KReWeight_xs',
                   'T2KReWeight_flux'], COLORS['T2KReWeight_flux'])
MASTER['cross-section'] = (['statistics_data',
                            'statistics_mc',
                            'T2KReWeight_fsi',
                            'T2KReWeight_xs'], COLORS['T2KReWeight_xs'])
MASTER['fsi'] = (['statistics_data',
                  'statistics_mc',
                  'T2KReWeight_fsi'], COLORS['T2KReWeight_fsi'])
MASTER['statistical (mc)'] = (['statistics_data',
                               'statistics_mc'], COLORS['statistics_mc'])
MASTER['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

STATSYST = ordereddict.OrderedDict()
STATSYST['systematics'] = (['statistics_data',
                            'statistics_mc',
                            'T2KReWeight_fsi',
                            'T2KReWeight_xs',
                            'T2KReWeight_flux',
                            'Mass_all',
                            'highland_all_syst'], COLORS['T2KReWeight_fsi'])
STATSYST['statistics'] = (['statistics_data',
                           'statistics_mc'], COLORS['statistics_data'])

STATONLY = ordereddict.OrderedDict()
STATONLY['statistical (mc)'] = (['statistics_data',
                                 'statistics_mc'], COLORS['statistics_mc'])
STATONLY['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

STATDATA = ordereddict.OrderedDict()
STATDATA['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

STATMC = ordereddict.OrderedDict()
STATMC['statistical (mc)'] = (['statistics_mc'], COLORS['statistics_mc'])

XSFSIMC = ordereddict.OrderedDict()
XSFSIMC['fsi'] = (['statistics_mc',
                   'T2KReWeight_xs',
                   'T2KReWeight_fsi'], COLORS['T2KReWeight_fsi'])
XSFSIMC['cross-section'] = (['statistics_mc',
                             'T2KReWeight_xs'], COLORS['T2KReWeight_xs'])
XSFSIMC['statistical (mc)'] = (['statistics_mc'], COLORS['statistics_mc'])

MODELSTAT = ordereddict.OrderedDict()
MODELSTAT['cross-section'] = (['statistics_data',
                               'statistics_mc',
                               'T2KReWeight_xs'], COLORS['T2KReWeight_xs'])
MODELSTAT['statistical (mc)'] = (['statistics_data',
                                 'statistics_mc'], COLORS['statistics_mc'])
MODELSTAT['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

FLUXSTAT = ordereddict.OrderedDict()
FLUXSTAT['flux'] = (['statistics_data',
                     'statistics_mc',
                     'T2KReWeight_flux'], COLORS['T2KReWeight_flux'])
FLUXSTAT['statistical (mc)'] = (['statistics_data',
                                 'statistics_mc'], COLORS['statistics_mc'])
FLUXSTAT['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

T2KRW = ordereddict.OrderedDict()
T2KRW['flux'] = (['T2KReWeight_fsi',
                  'T2KReWeight_xs',
                  'T2KReWeight_flux'], COLORS['T2KReWeight_flux'])
T2KRW['cross-section'] = (['T2KReWeight_fsi',
                           'T2KReWeight_xs'], COLORS['T2KReWeight_xs'])
T2KRW['fsi'] = (['T2KReWeight_fsi'], COLORS['T2KReWeight_fsi'])

T2KFLUX = ordereddict.OrderedDict()
T2KFLUX['flux'] = (['T2KReWeight_flux'], COLORS['T2KReWeight_flux'])

T2KFSI = ordereddict.OrderedDict()
T2KFSI['fsi'] = (['T2KReWeight_fsi'], COLORS['T2KReWeight_fsi'])

T2KXS = ordereddict.OrderedDict()
T2KXS['cross-section'] = (['T2KReWeight_xs'], COLORS['T2KReWeight_xs'])

DETECTOR = ordereddict.OrderedDict()
DETECTOR['detector'] = (['highland_all_syst'], COLORS['detector'])

MASS = ordereddict.OrderedDict()
MASS['mass'] = (['Mass_all'], COLORS['Mass_all'])

OOFV = ordereddict.OrderedDict()
OOFV['oofv'] = (['highland_p0dccqeoofv_syst'], COLORS['detector'])

# for merged flux, xs, fsi throws
ALL = ordereddict.OrderedDict()
ALL['detector'] = (['statistics_data',
                    'statistics_mc',
                    'T2KReWeight_all',
                    'Mass_all',
                    'highland_all_syst'], COLORS['detector'])
ALL['mass'] = (['statistics_data',
                'statistics_mc',
                'T2KReWeight_all',
                'Mass_all'], COLORS['Mass_all'])
ALL['theory'] = (['statistics_data',
                  'statistics_mc',
                  'T2KReWeight_all'], COLORS['T2KReWeight_fsi'])
ALL['statistical (mc)'] = (['statistics_data',
                               'statistics_mc'], COLORS['statistics_mc'])
ALL['statistical (data)'] = (['statistics_data'], COLORS['statistics_data'])

# for ungrouped t2krw dials
UNGROUPEDXS = ordereddict.OrderedDict()
UNGROUPEDXS['MaQE'] = (['T2KReWeight_NIWG2014a_pF_O16',
                        'T2KReWeight_NIWG2014a_Eb_O16',
                        'T2KReWeight_NIWGMEC_Norm_O16',
                        'T2KReWeight_NXSec_CA5RES',
                        'T2KReWeight_NXSec_MaNFFRES',
                        'T2KReWeight_NXSec_MaCCQE'], COLORS['T2KReWeight_xs'])
UNGROUPEDXS['MaNFFRES'] = (['T2KReWeight_NIWG2014a_pF_O16',
                        'T2KReWeight_NIWG2014a_Eb_O16',
                        'T2KReWeight_NIWGMEC_Norm_O16',
                        'T2KReWeight_NXSec_CA5RES',
                        'T2KReWeight_NXSec_MaNFFRES'], COLORS['T2KReWeight_fsi'])
UNGROUPEDXS['CA5RES'] = (['T2KReWeight_NIWG2014a_pF_O16',
                        'T2KReWeight_NIWG2014a_Eb_O16',
                        'T2KReWeight_NIWGMEC_Norm_O16',
                        'T2KReWeight_NXSec_CA5RES'], COLORS['T2KReWeight_flux'])
UNGROUPEDXS['MEC_O16'] = (['T2KReWeight_NIWG2014a_pF_O16',
                        'T2KReWeight_NIWG2014a_Eb_O16',
                        'T2KReWeight_NIWGMEC_Norm_O16'], COLORS['detector'])
UNGROUPEDXS['Eb_O16'] = (['T2KReWeight_NIWG2014a_pF_O16',
                        'T2KReWeight_NIWG2014a_Eb_O16'], COLORS['statistics_mc'])
UNGROUPEDXS['pF_O16'] = (['T2KReWeight_NIWG2014a_pF_O16'], COLORS['statistics_data'])

MAQE = ordereddict.OrderedDict()
MAQE['MaQE'] = (['T2KReWeight_NXSec_MaCCQE'], COLORS['T2KReWeight_xs'])
EBO = ordereddict.OrderedDict()
EBO['Eb_O16'] = (['T2KReWeight_NIWG2014a_Eb_O16'], COLORS['statistics_mc'])
PFO = ordereddict.OrderedDict()
PFO['pF_O16'] = (['T2KReWeight_NIWG2014a_pF_O16'], COLORS['statistics_data'])
MECO = ordereddict.OrderedDict()
MECO['MEC_O16'] = (['T2KReWeight_NIWGMEC_Norm_O16'], COLORS['detector'])
##################### End #######################

#################### Volumes ####################
DEFAULT_VOLUME = 'Entire P0D-FV'
XLAYER_VOLUME = 'X-layer Only'
YLAYER_VOLUME = 'Y-layer Only'
BAGXLAYER_VOLUME = 'No Y-layers'
##################### End #######################

################### Reweight ####################
BANFF_FILE = os.path.join(directories.IRODS,
                          'asg/banff/postfit/postfit_banff_2015_data_20150417_allparams.root')
MASS_FILE = os.path.join(os.getenv('P0DNUMUCCANALYSISROOT'), 'data', 'P0DMassWeights.dat')
EMC_FILE = os.path.join(directories.WORK314, 'myAnalyses', 'P0DCC0Pi', 'Systematics', 'emc.root')
##################### End #######################

################### xsCache #####################
XSCACHE_FILE = os.path.join(directories.XSCACHE_STORE, 'xscache.root')
##################### End #######################

############# flux test files ###################
# these files were generated with 5000 throws of the t2kreweight
# parameters using a small highland2 microtree. this allows us to get
# a large number of throws to calculate the flux uncertainties.
JEREMY_TEST = '/usr/users/jplopez/data41/T2K/Work/P0DCC0Pi/FluxTest/'
MCP_TEST = os.path.join(JEREMY_TEST, 'microtree.root')
RDP_TEST = os.path.join(JEREMY_TEST, 'microtree.root')
WEIGHTS_TEST = os.path.join(JEREMY_TEST, 'out2', 'weights_unified.root')
RWFILE_DEFAULT_TEST = os.path.join(JEREMY_TEST, 'out2', 'rwfile_default.root')
RWFILE_TRUTH_TEST = os.path.join(JEREMY_TEST, 'out2', 'rwfile_truth.root')
##################### End #######################

### DEBUG ###
if DEBUG:
    # Some alternative selections
    SELECT_XLAYER = SELECTIONPAIR(SELECT.reco+'&&'+getLayerCutStr('selmu_pos[2]', 'x'),
                                  SELECT.true+'&&'+getLayerCutStr('truemu_pos[2]', 'x'))
    SELECT_YLAYER = SELECTIONPAIR(SELECT.reco+'&&'+getLayerCutStr('selmu_pos[2]', 'y'),
                                  SELECT.true+'&&'+getLayerCutStr('truemu_pos[2]', 'y'))
    SELECT_BAGX = SELECTIONPAIR(SELECT.reco+'&&'+getLayerCutStr('selmu_pos[2]', 'bagx'),
                                SELECT.true+'&&'+getLayerCutStr('truemu_pos[2]', 'bagx'))

    # testing 100% pure sideband
    PURE_SIDEBANDS = [SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d == 2 && topology==1', 'topology == 1'),
                      SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d > 2 && topology==2', 'topology == 2')]

    MICHEL1_SIDEBANDS = [SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d == 2 && nmichel==1', 'topology == 1'),
                         SELECTIONPAIR('Sum$(accum_level[0][1]) > 3 && np0d > 2', 'topology == 2')]

    # testing sideband with additiona dphiT cut
    DPHIT_SIDEBANDS = [SELECTIONPAIR(
        'Sum$(accum_level[0][1]) > 3 && np0d == 2 && sel_dphi*180/TMath::Pi()>60', 'topology == 1')]

    # debugging source error dependece on the order of sources, fixed due to bincache
    # DETECTOR['detector'] = (['T2KReWeight_fsi', 'highland_all_syst'], COLORS['detector'])


import falcon

from sqlalchemy.orm import Session
from sqlalchemy import exists, and_
from database import engine, Base
from tables import UsersTable, SignaturesTable
from time import ctime

from utilities import getSignatureQuery, getJson, to_bool
from auth import authenticate

def recordUser(session, valueDict):
        userRow = None
        requiredValues = ["Signature", "Name", "Email", "UserID"]

        for value in requiredValues:
            if value not in valueDict.keys():
                return None
                #resp.body = "Error, missing key {}".format(value)
                #resp.status = falcon.HTTP_400

        # User table values
        signature = valueDict.get("Signature")
        name = valueDict.get("Name")
        email = valueDict.get("Email")
        company = valueDict.get("Company")
        userID = valueDict.get("UserID")

        userExists = True
        userRow = session.query(UsersTable).get((signature, userID))

        if userRow is None:
            userExists = False
            userRow = UsersTable(Signature=signature, Name=name, Email=email, Company=company, UserID=userID, InstallDateTime=ctime())

        return userRow, userExists

def createSignatureRow(session, valueDict):
        signatureRow = None
        signature = valueDict.get("Signature")

        # Signature table values
        installLimit = valueDict.get("InstallLimit")
        unlimitedInstalls = to_bool(valueDict.get("UnlimitedInstalls"))

        if unlimitedInstalls is True:
            signatureRow = SignaturesTable(PrimaryKey=signature, InstallCount=0, InstallLimit=0, UnlimitedInstalls=unlimitedInstalls)
            print("Created Signature: {}, with UnlimitedInstalls".format(signature))
        elif installLimit is not None:
            signatureRow = SignaturesTable(PrimaryKey=signature, InstallCount=0, InstallLimit=installLimit, UnlimitedInstalls=False)
            print("Created Signature: {}, with InstallLimit of: {}".format(signature, installLimit))
        else:
            print("Error creating row with Signature: {}. No valid UnlimitedInstalls or InstallLimit provided".format(signature))

        return signatureRow

class Resource(object):

    def on_get(self, req, resp):
        resp.body = "Accepted!"
        resp.status = falcon.HTTP_200

    def on_post(self, req, resp):
        validRequest = authenticate(req)

        if not validRequest:
            resp.body = "Invalid username/password"
            resp.status = falcon.HTTP_401
            return

        session = Session(engine)
        valueDict = getJson(req)

        signatureQuery = getSignatureQuery(req, session)

        message = "Unable to add Signature"
        resp.status = falcon.HTTP_400

        if "Signature" in valueDict.keys() and signatureQuery is None:
            signatureRow = createSignatureRow(session, valueDict)
            message = "Unable to create signature row"
            if signatureRow is not None:
                session.add(signatureRow)
                message = "Added signature to database: {}".format(signatureRow.PrimaryKey)
                resp.status = falcon.HTTP_200

        elif "Signature" in valueDict.keys():
            message = "Unable to add Signature, already exists in database"

        resp.body = message
        print(message)

        session.commit()
        session.close()


# -*- coding = utf-8 -*-
# @Author: youngfuns
# @Time : 2021/12/11 21:49
# @File : demo1.py
# @Software: PyCharm
import re
import urllib.error  # 制定url， 获取网页数据
import urllib.request
from bs4 import BeautifulSoup
import xlwt  # 进行Excel操作
import sqlite3

def main():
    baseurl = "https://movie.douban.com/top250?start="
    # 爬取网页
    datalist = getData(baseurl)
    # 保存到Excel
    # savepath = '.\\豆瓣电影top250.xls'
    # saveData(datalist, savepath)
    #  保存到SQLit
    sqlitePath = 'movie250.db'
    saveToDb(datalist, sqlitePath)

findLink = re.compile(r'')
findImgSrc = re.compile(r'', re.S)
findTitle = re.compile(r'(.*)')
findRating = re.compile(r'span class="rating_num" property="v:average">(.*)')
findJudge = re.compile(r'(\d*)人评价')
findInq = re.compile(r'span class="inq">(.*)')
findBd = re.compile(r'
(.*?)
', re.S)

# 爬取网页
def getData(baseurl):
    datalist = []
    for i in range(0, 10):
        url = baseurl + str(i * 25)
        html = askUrl(url)
        # print(url)

        # 解析数据
        soup = BeautifulSoup(html, "html.parser")
        # print(soup.findAll("a"))
        for item in soup.findAll("div", class_="item"):
            data = []
            item = str(item)

            # 获取相关信息
            link = re.findall(findLink, item)[0]
            data.append(link)

            imgSrc = re.findall(findImgSrc, item)[0]
            print(re.findall(findImgSrc, item))
            data.append(imgSrc)

            titles = re.findall(findTitle, item)
            if len(titles) == 2:
                ctitle = titles[0]
                data.append(ctitle)
                otitle = titles[1].replace("/", "")
                data.append(otitle)
            else:
                data.append(titles[0])
                data.append(" ")

            rating = re.findall(findRating, item)[0]
            data.append(rating)

            judgeNum = re.findall(findJudge, item)[0]
            data.append(judgeNum)

            inq = re.findall(findInq, item)
            if len(inq) != 0:
                inq = inq[0].replace("。", "")
                data.append(inq)
            else:
                data.append(" ")

            bd = re.findall(findBd, item)[0]
            bd = re.sub('
(\s+)?', ' ', bd)
            bd = re.sub('/', " ", bd)
            data.append(bd.strip())

            datalist.append(data)

    # print(datalist)
    return datalist

def askUrl(url):
    head = {
        "user-agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 96.0.4664.93Safari / 537.36"
    }
    request = urllib.request.Request(url, headers=head)
    # html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
        return html
        # print(html)
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
            if hasattr(e, "reason"):
                print(e.reason)

# 保存数据
def saveData(datalist, savepath):
    book = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = book.add_sheet("豆瓣电影TOP250", cell_overwrite_ok=True)
    col = ("电影详情链接", "图片链接", "影片中文名", "影片外国名", "评分", "评价数", "概况", "相关信息")
    for i in range(0, 8):
        sheet.write(0, i, col[i])
    for i in range(0, 250):
        print("爬取第%d条" % i)
        for j in range(0, 8):
            sheet.write(i + 1, j, datalist[i][j])
    book.save(savepath)

def saveToDb(datalist, dbpath):
    # 连接数据库 创建数据表
    init_db(dbpath)
    # 插入数据
    conn = sqlite3.connect(dbpath)
    cursor = conn.cursor()
    for data in datalist:
        for i in data:
            if data.index(i) == 4 or data.index(i) == 5:
                continue
                i = '"' + i + '"'
        sql = '''
            insert movie_top (
                info_link,
                banner_link,
                title,
                i_title,
                rating,
                judge,
                inq,
                info
            ) values (%s);
        ''' % ','.join(data)
        # cursor.execute(sql)
        # conn.commit()
        # conn.close()
        print(sql)
    # info = sqlite3.connect(dbpath)
    # cu = info.cursor()
    # c = cu.execute('select * from movie_top')
    # print(c)
    # info.commit()
    # info.close()

def init_db(dbpath):
    sql = '''
        create table movie_top(
        id integer primary key autoincrement,
        info_link text,
        banner_link text,
        title varchar,
        i_title varchar,
        rating numeric,
        judge numeric,
        inq text,
        info text
        )
    '''

    conn = sqlite3.connect(dbpath)
    cursor = conn.cursor()
    cursor.execute(sql)
    conn.commit()
    conn.close()

if __name__ == "__main__":
    main()


class Bstnode:
    def __init__(self, newkey, newval):
        self.key = newkey
        self.value = newval
        self.left = None
        self.right = None

class BST:
    def __init__(self):
        self.root = None
        self.size = 0

    def length(self):
        return self.size

    def bstsearch(self, subtree, target):
        if subtree is None:
            return None
        elif target > subtree.key:
            return self.bstsearch(subtree.right, target)
        elif target < subtree.key:
            return self.bstsearch(subtree.left, target)
        else:
            return subtree

    def has_key(self, target):
        return self.bstsearch(self.root, target) is not None

    def valueof(self, target):
        node = self.bstsearch(self.root, target)
        assert node is not None, "Invalid key"
        return node.value

    def bstminimum(self, subtree):
        if subtree is None:
            return None
        elif subtree.left is None:
            return subtree
        else:
            return self.bstminimum(subtree.left)

    def bstmaximum(self, subtree):
        if subtree is None:
            return None
        elif subtree.right is None:
            return subtree
        else:
            return self.bstminimum(subtree.right)

    def add(self, newkey, newval):
        node = self.bstsearch(self.root, newkey)
        if node is not None:
            node.value = newval
            return False
        else:
            self.root = self.bstinsert(self.root, newkey, newval)
            self.size += 1
            return True

    def bstinsert(self, subtree, newkey, newval):
        if subtree is None:
            subtree = Bstnode(newkey, newval)
        elif newkey > subtree.key:
            subtree.right = self.bstinsert(subtree.right, newkey, newval)
        else:
            subtree.left = self.bstinsert(subtree.left, newkey, newval)
        return subtree

    def remove(self, key):
        node = self.bstsearch(self.root, key)
        assert node is not None, "Invalid key"
        self.root = self.bstremove(self.root, key)
        self.size -= 1

    def bstremove(self, subtree, key):
        if key > subtree.key:
            subtree.right = self.bstremove(subtree.right, key)
        elif key < subtree.key:
            subtree.left = self.bstremove(subtree.left, key)
        else:
            if subtree.left is None and subtree.right is None:
                return None
            elif subtree.left is not None and subtree.right is None:
                return subtree.left
            elif subtree.left is None and subtree.right is not None:
                return subtree.right
            else:
                successor = self.bstminimum(subtree.right)
                subtree.key = successor.key
                subtree.value = successor.value
                subtree.right = self.bstremove(subtree.right, successor.key)
                return subtree


import sys

import tiktoken
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import UnstructuredPDFLoader
from langchain.schema import HumanMessage, SystemMessage

INPUT_PRICE_PER_1K_TOKENS = 0.0015
OUTPUT_PRICE_PER_1K_TOKENS = 0.002
YEN_PER_DOLLAR = 140

MODEL_NAME = "gpt-3.5-turbo"

SYSTEM_PROMPT = """Please translate the following English text into Japanese.
Please output in markdown format.

Input is extracted from PDF, so it may contain elements other than the title and text.
Please ignore elements that do not appear to be titles and body text.

日本語でお願いします。
"""

def translate(text: str, callback) -> str:
    chat = ChatOpenAI(
        model_name=MODEL_NAME,
        temperature=0,
        streaming=True,
        callbacks=[callback],
    )

    messages = [
        SystemMessage(content=SYSTEM_PROMPT),
        HumanMessage(content=text),
    ]

    return chat(messages).content

def count_tokens(text):
    encoding = tiktoken.encoding_for_model(MODEL_NAME)
    tokens = encoding.encode(text)
    return len(tokens)

def calculate_price_as_doller(input_token_count, output_token_count):
    return (
        input_token_count * INPUT_PRICE_PER_1K_TOKENS / 1000
        + output_token_count * OUTPUT_PRICE_PER_1K_TOKENS / 1000
    )

def doller_to_yen(doller):
    return doller * YEN_PER_DOLLAR

def load_pdf(file_path: str) -> list[str]:
    loader = UnstructuredPDFLoader(file_path=file_path, mode="elements")
    docs = loader.load()

    last_doc = docs[-1]
    last_doc_page_number = last_doc.metadata["page_number"]

    texts = [""] * last_doc_page_number

    for doc in docs:
        metadata = doc.metadata
        page_number = metadata["page_number"]
        category = metadata["category"]

        if category in ["Title", "NarrativeText"]:
            texts[page_number - 1] += doc.page_content + "\n\n"
        elif category in ["UncategorizedText", "ListItem"]:
            pass
        else:
            raise ValueError(f"Unknown category: {category}")

    return texts

if __name__ == "__main__":
    file_path = sys.argv[1]
    texts = load_pdf(file_path)
    for i, text in enumerate(texts):
        print(f"========== Page {i+1} ==========")
        print(text)


# Day 16: Ticket Translation

INPUT_FILE_NAME = "./inputs/day16input.txt"
TEST_FILE_NAME = "./inputs/day16testinput.txt"
TEST_FILE_NAME_2 = "./inputs/day16testinput2.txt"

def get_valid_numbers(interval_string):
    '''
    Returns a list of valid numbers given an interval in string
    notations.
    '''
    start, end = tuple(int(num) for num in interval_string.split('-'))
    return [val for val in range(start, end + 1)]

def parse_rule(line):
    '''
    Returns a dictionary of the field mapped to its valid values.
    '''
    field, intervals = line.split(': ')
    interval_one, interval_two = intervals.split(' or ')

    valid_numbers = get_valid_numbers(interval_one)
    valid_numbers.extend(get_valid_numbers(interval_two))
    return {field: valid_numbers}, set(valid_numbers)

def read_file(file_nm):
    '''
    Returns dictionary of rules where the field is mapped to its
    valid values, list of all valid vlaues, your ticket information,
    and nearby tickets information.
    '''
    rules_section = True
    your_ticket_section = False
    nearby_tickets_section = False
    rules = {}
    valid_numbers = set()
    your_ticket = []
    nearby_tickets = []

    file = open(file_nm, 'r')
    for line in file:
        line = line.strip('\n')
        if not line:
            continue
        elif 'your ticket' in line:
            your_ticket_section = True
            rules_section = False
        elif 'nearby tickets' in line:
            nearby_tickets_section = True
            your_ticket_section = False
        elif rules_section:
            rule, valid_numbers_subset = parse_rule(line)
            rules.update(rule)
            valid_numbers.update(valid_numbers_subset)
        else:
            # turn line into tuple of numbers
            ticket = tuple(int(num) for num in line.split(','))
            if your_ticket_section:
                your_ticket = ticket
            else:
                nearby_tickets.append(ticket)
    file.close()

    return rules, valid_numbers, your_ticket, nearby_tickets

def is_valid_ticket(nearby_ticket, valid_set):
    '''
    Checks if nearby ticket is a valid ticket
    '''
    for value in nearby_ticket:
        if value not in valid_set:
            return False
    return True

def get_invalid_nums(nearby_tickets, valid_set):
    '''
    Returns list of invalid numbers of all the nearby tickets.
    '''
    invalid_numbers = []
    for ticket in nearby_tickets:
        for value in ticket:
            if value not in valid_set:
                invalid_numbers.append(value)
    return invalid_numbers

def filter_nearby_tickets(nearby_tickets, valid_set):
    '''
    Filters and returns an updated list of nearby tickets.
    '''
    updated_nearby_tickets = []
    for ticket in nearby_tickets:
        if is_valid_ticket(ticket, valid_set):
            updated_nearby_tickets.append(ticket)
    return updated_nearby_tickets

def get_valid_values_per_field(nearby_tickets):
    '''
    Given a list of valid nearby tickets, returns a list of lists
    where each list contains all valid values for one field collectively. Fields
    are in the same order for all tickets.
    '''
    valid_fields_values = [[] for field in range(len(nearby_tickets[0]))]
    for ticket in nearby_tickets:
        for i in range(len(ticket)):
            valid_fields_values[i].append(ticket[i])

    return valid_fields_values

def match_fields(rules, valid_fields_values):
    '''
    Returns a dictionary of fields mapped to their positions (index) on a ticket.
    '''
    fields_key = {}
    for field in rules:
        fields_key[field] = []

    for index in range(len(valid_fields_values)):
        valid_sub_lst = valid_fields_values[index]
        for field in rules:
            could_be_field = True
            for number in valid_sub_lst:
                if number not in rules[field]:
                    could_be_field = False
                    break
            if could_be_field:
                fields_key[field].append(index)

    index_confirmed = set()
    while len(index_confirmed) != len(fields_key):
        index = None
        for field in fields_key:
            if len(fields_key[field]) == 1 and fields_key[field][0] not in index_confirmed:
                index = fields_key[field][0]
                break

        for field in fields_key:
            if len(fields_key[field]) == 1:
                continue
            if index in fields_key[field]:
                fields_key[field].remove(index)

        index_confirmed.add(index)

    return fields_key

def retrieve_index_list(fields_dict, sub_key_phrase):
    '''
    Returns list of positions of fields containing a subphrase.
    '''
    index_lst = []
    for field in fields_dict:
        if sub_key_phrase in field:
            index_lst.extend(fields_dict[field])
    return index_lst

def solve_part_one(nearby_tickets, valid_numbers):
    '''
    Returns sum of all invalid values, excluding your ticket.
    '''
    return sum(get_invalid_nums(nearby_tickets, valid_numbers))

def solve_part_two(rules, valid_numbers, your_ticket, nearby_tickets):
    '''
    Returns sum of all invalid values, excluding your ticket.
    '''
    nearby_tickets = filter_nearby_tickets(nearby_tickets, valid_numbers)
    valid_fields_values = get_valid_values_per_field(nearby_tickets)
    fields_info = match_fields(rules, valid_fields_values)
    departure_list = retrieve_index_list(fields_info, 'departure')

    acc = 1
    for elem in departure_list:
        acc *= your_ticket[elem]
    return acc

def run_test_file(file_nm):
    '''
    Returns field positions given a test file
    '''
    rules, valid_numbers, your_ticket, nearby_tickets = read_file(file_nm)
    nearby_tickets = filter_nearby_tickets(nearby_tickets, valid_numbers)
    valid_fields_values = get_valid_values_per_field(nearby_tickets)
    fields_info = match_fields(rules, valid_fields_values)
    return fields_info

def main():
    test_rules, test_valid_numbers, test_your_ticket, test_nearby_tickets = read_file(TEST_FILE_NAME)
    assert(solve_part_one(test_nearby_tickets, test_valid_numbers) == 71)
    assert(run_test_file(TEST_FILE_NAME_2) == {'row': [0], 'class': [1], 'seat': [2]})

    rules, valid_numbers, your_ticket, nearby_tickets = read_file(INPUT_FILE_NAME)
    print('Part One:', solve_part_one(nearby_tickets, valid_numbers))
    print('Part Two:', solve_part_two(rules, valid_numbers, your_ticket, nearby_tickets))

main()


import asyncio
import contextlib
import functools
import discord
from datetime import datetime, timedelta, timezone
from typing import Tuple, Optional, List, no_type_check
from redbot.core import commands, checks
from redbot.core.config import Config
from redbot.core.i18n import Translator, cog_i18n
from redbot.core.utils.menus import menu, DEFAULT_CONTROLS

from .message import SchedulerMessage
from .logs import get_logger
from .tasks import Task
from .converters import Schedule, non_numeric, TempMute
from .checks import can_run_command

_ = Translator("And I think it's gonna be a long long time...", __file__)

@cog_i18n(_)
class Scheduler(commands.Cog):
    """
    A somewhat sane scheduler cog
    """

    __version__ = "1.0.27"
    __author__ = "mikeshardmind(Sinbad), DiscordLiz"
    __flavor_text__ = "Unhidden remindme."

    def __init__(self, bot):
        self.bot = bot
        self.config = Config.get_conf(
            self, identifier=78631113035100160, force_registration=True
        )
        self.config.register_channel(tasks={})  # Serialized Tasks go in here.
        self.log = get_logger("sinbadcogs.scheduler")
        self.bg_loop_task = bot.loop.create_task(self.bg_loop())
        self.scheduled = {}  # Might change this to a list later.
        self.tasks = []
        self._iter_lock = asyncio.Lock()

    def cog_unload(self):
        self.bg_loop_task.cancel()
        for task in self.scheduled.values():
            task.cancel()
        self.log.handlers = []

    # This never should be needed,
    # but it doesn't hurt to add and could cover a weird edge case.
    __del__ = cog_unload

    async def _load_tasks(self):
        chan_dict = await self.config.all_channels()
        for channel_id, channel_data in chan_dict.items():
            channel = self.bot.get_channel(channel_id)
            if (
                not channel
                or not channel.permissions_for(channel.guild.me).read_messages
            ):
                continue
            tasks_dict = channel_data.get("tasks", {})
            for t in Task.bulk_from_config(bot=self.bot, **tasks_dict):
                self.tasks.append(t)

    async def _remove_tasks(self, *tasks: Task):
        async with self._iter_lock:
            for task in tasks:
                self.tasks.remove(task)
                await self.config.channel(task.channel).clear_raw("tasks", task.uid)

    async def bg_loop(self):
        await self.bot.wait_until_ready()
        await asyncio.sleep(2)
        _guilds = [
            g for g in self.bot.guilds if g.large and not (g.chunked or g.unavailable)
        ]
        await self.bot.request_offline_members(*_guilds)

        async with self._iter_lock:
            await self._load_tasks()
        while self is self.bot.get_cog("Scheduler"):
            sleep_for = await self.schedule_upcoming()
            await asyncio.sleep(sleep_for)

    async def delayed_wrap_and_invoke(self, task: Task, delay: int):
        await asyncio.sleep(delay)
        task.update_objects(self.bot)
        chan = task.channel
        if not chan.permissions_for(chan.guild.me).read_messages:
            return
        message = await task.get_message(self.bot)
        context = await self.bot.get_context(message)
        context.assume_yes = True
        await self.bot.invoke(context)
        for cog_name in ("CustomCommands", "Alias"):
            cog = self.bot.get_cog(cog_name)
            if cog:
                await cog.on_message(message)
        # TODO: allow registering additional cogs to process on_message for.

    async def schedule_upcoming(self) -> int:
        """
        Schedules some upcoming things as tasks.

        """

        # TODO: improve handlng of next time return

        async with self._iter_lock:
            to_pop = []
            for k, v in self.scheduled.items():
                if v.done():
                    to_pop.append(k)
                    try:
                        v.result()
                    except Exception:
                        self.log.exception("Dead task ", exc_info=True)

            for k in to_pop:
                self.scheduled.pop(k, None)

        to_remove: list = []

        for task in self.tasks:
            delay = task.next_call_delay
            if delay < 30 and task.uid not in self.scheduled:
                self.scheduled[task.uid] = asyncio.create_task(
                    self.delayed_wrap_and_invoke(task, delay)
                )
                if not task.recur:
                    to_remove.append(task)

        await self._remove_tasks(*to_remove)

        return 15

    @property
    def task_class(self):
        """ API class Access """
        return Task

    async def submit_task(self, task: Task):
        """ externally safe API """
        async with self._iter_lock:
            self.tasks.append(task)

    async def fetch_task_by_attrs_exact(self, **kwargs) -> List[Task]:
        def pred(item):
            try:
                return kwargs and all(getattr(item, k) == v for k, v in kwargs.items())
            except AttributeError:
                return False

        async with self._iter_lock:
            return [t for t in self.tasks if pred(t)]

    async def fetch_task_by_attrs_lax(
        self, lax: Optional[dict] = None, strict: Optional[dict] = None
    ) -> List[Task]:
        def pred(item):
            try:
                if strict:
                    if not all(getattr(item, k) == v for k, v in strict.items()):
                        return False
            except AttributeError:
                return False
            if lax:
                return any(getattr(item, k, None) == v for k, v in lax.items())
            return True

        async with self._iter_lock:
            return [t for t in self.tasks if pred(t)]

    async def fetch_tasks_by_guild(self, guild: discord.Guild) -> List[Task]:

        async with self._iter_lock:
            return [t for t in self.tasks if t.channel in guild.text_channels]

    # Commands go here

    @checks.mod_or_permissions(manage_guild=True)
    @commands.guild_only()
    @commands.command(usage="<eventname> <command> <args>")
    @no_type_check
    async def schedule(self, ctx, event_name: non_numeric, *, schedule: Schedule):
        """
        Schedule something

        Usage:
            [p]schedule eventname command args

        args:

            you must provide one of:

                --start-in interval
                --start-at time

            you may also provide:

                --every interval

            for recurring tasks

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.

        Example use:

            [p]schedule autosync bansync True --start-at 12AM --every 1 day

        Example use with other parsed commands:

        [p]schedule autosyndicate syndicatebans --sources 133049272517001216 --auto-destinations -- --start-at 12AM --every 1 hour

        This can also execute aliases.
        """
        schedule: Tuple[str, datetime, Optional[timedelta]]

        command, start, recur = schedule

        t = Task(
            uid=ctx.message.id,
            nicename=event_name,
            author=ctx.author,
            content=command,
            channel=ctx.channel,
            initial=start,
            recur=recur,
        )

        if await self.fetch_task_by_attrs_exact(
            author=ctx.author, channel=ctx.channel, nicename=event_name
        ):
            return await ctx.send("You already have an event by that name here.")

        async with self._iter_lock:
            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(t.to_config())
            self.tasks.append(t)

        ret = (
            f"Task Scheduled. You can cancel this task with "
            f"`{ctx.clean_prefix}unschedule {ctx.message.id}` "
            f"or with `{ctx.clean_prefix}unschedule {event_name}`"
        )

        if recur and t.next_call_delay < 60:
            ret += (
                "\nWith the intial start being set so soon, "
                "you might have missed an initial use being scheduled by the loop. "
                "you may find the very first expected run of this was missed or otherwise seems late. "
                "Future runs will be on time."  # fractions of a second in terms of accuracy.
            )

        await ctx.send(ret)

    @commands.guild_only()
    @commands.command()
    async def unschedule(self, ctx, info):
        """
        unschedule something.
        """

        tasks = await self.fetch_task_by_attrs_lax(
            lax={"uid": info, "nicename": info},
            strict={"author": ctx.author, "channel": ctx.channel},
        )

        if not tasks:
            await ctx.send(
                f"Hmm, I couldn't find that task. (try `{ctx.clean_prefix}showscheduled`)"
            )

        elif len(tasks) > 1:
            self.log.warning(
                f"Mutiple tasks where should be unique. Task data: {tasks}"
            )
            return await ctx.send(
                "There seems to have been breakage here. Cleaning up and logging incident."
            )

        else:
            await self._remove_tasks(*tasks)
            await ctx.tick()

    @commands.guild_only()
    @commands.command()
    async def showscheduled(self, ctx: commands.Context, all_channels: bool = False):
        """ shows your scheduled tasks in this, or all channels """

        if all_channels:
            tasks = await self.fetch_tasks_by_guild(ctx.guild)
            tasks = [t for t in tasks if t.author == ctx.author]
        else:
            tasks = await self.fetch_task_by_attrs_exact(
                author=ctx.author, channel=ctx.channel
            )

        if not tasks:
            return await ctx.send("No scheduled tasks")

        await self.task_menu(ctx, tasks)

    async def task_menu(self, ctx, tasks, message: Optional[discord.Message] = None):

        color = await ctx.embed_color()

        async def task_killer(
            cog: "Scheduler",
            page_mapping: dict,
            ctx: commands.Context,
            pages: list,
            controls: dict,
            message: discord.Message,
            page: int,
            timeout: float,
            emoji: str,
        ):
            to_cancel = page_mapping.pop(page)
            await cog._remove_tasks(to_cancel)
            if page_mapping:
                tasks = list(page_mapping.values())
                await cog.task_menu(ctx, tasks, message)
            else:
                with contextlib.suppress(discord.NotFound):
                    await message.delete()

        count = len(tasks)
        embeds = [
            t.to_embed(index=i, page_count=count, color=color)
            for i, t in enumerate(tasks, 1)
        ]

        controls = DEFAULT_CONTROLS.copy()
        page_mapping = {i: t for i, t in enumerate(tasks)}
        actual_task_killer = functools.partial(task_killer, self, page_mapping)
        controls.update({"\N{NO ENTRY SIGN}": actual_task_killer})
        await menu(ctx, embeds, controls, message=message)

    @commands.command(name="remindme", usage="<what to be reminded of> <args>")
    async def reminder(self, ctx, *, reminder: Schedule):
        """
        Schedule a reminder DM from the bot

        Usage:
            [p]remindme to do something [args]

        args:

            you must provide one of:

                --start-in interval
                --start-at time

            you may also provide:

                --every interval

            for recurring reminders

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.

        example usage:
            `[p]remindme get some fresh air --start-in 4 hours`
        """

        command, start, recur = reminder

        t = Task(
            uid=ctx.message.id,
            nicename=f"reminder-{ctx.message.id}",
            author=ctx.author,
            content=f"schedhelpers selfwhisper {command}",
            channel=ctx.channel,
            initial=start,
            recur=recur,
        )

        async with self._iter_lock:
            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(t.to_config())
            self.tasks.append(t)

        await ctx.tick()

    @commands.check(lambda ctx: ctx.message.__class__.__name__ == "SchedulerMessage")
    @commands.group(hidden=True, name="schedhelpers")
    async def helpers(self, ctx):
        """ helper commands for scheduler use """
        pass

    @helpers.command(name="say")
    async def say(self, ctx, *, content):
        await ctx.send(content)

    @helpers.command(name="selfwhisper")
    async def swhisp(self, ctx, *, content):
        with contextlib.suppress(discord.HTTPException):
            await ctx.author.send(content)

    @commands.admin_or_permissions(manage_guild=True)
    @commands.guild_only()
    @commands.group()
    async def scheduleradmin(self, ctx):
        """ Administrative commands for scheduler """
        pass

    @scheduleradmin.command()
    async def viewall(self, ctx):
        """ view all scheduled events in a guild """

        tasks = await self.fetch_tasks_by_guild(ctx.guild)

        if not tasks:
            return await ctx.send("No scheduled tasks")

        await self.task_menu(ctx, tasks)

    @scheduleradmin.command()
    async def kill(self, ctx, *, task_id):
        """ kill another user's task (id only) """

        tasks = await self.fetch_task_by_attrs_exact(uid=task_id)

        if not tasks:
            await ctx.send(
                f"Hmm, I couldn't find that task. (try `{ctx.clean_prefix}showscheduled`)"
            )

        elif len(tasks) > 1:
            self.log.warning(
                f"Mutiple tasks where should be unique. Task data: {tasks}"
            )
            return await ctx.send(
                "There seems to have been breakage here. Cleaning up and logging incident."
            )

        else:
            await self._remove_tasks(*tasks)
            await ctx.tick()

    @scheduleradmin.command()
    async def killchannel(self, ctx, channel: discord.TextChannel):
        """ kill all in a channel """

        tasks = await self.fetch_task_by_attrs_exact(channel=channel)

        if not tasks:
            return await ctx.send("No scheduled tasks in that channel.")

        await self._remove_tasks(*tasks)
        await ctx.tick()

    @commands.guild_only()
    @commands.command(aliases=["tm"])
    @can_run_command("mute server")
    async def tempmute(self, ctx, user: discord.Member, *, mute: TempMute):
        """
        binding for mute + scheduled unmute
        This exists only until it is added to core red

        args can be
            --until time
        or
            --for interval

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.
        """

        reason, unmute_time = mute

        now = datetime.now(timezone.utc)

        mute_task = Task(
            uid=f"mute-{ctx.message.id}",
            nicename=f"mute-{ctx.message.id}",
            author=ctx.author,
            content=f"mute server {user.id} {reason}",
            channel=ctx.channel,
            initial=now,
            recur=None,
        )

        mute_task_2 = Task(
            uid=f"muterole-{ctx.message.id}",
            nicename=f"muterole-{ctx.message.id}",
            author=ctx.author,
            content=f"addrole nadeko-mute {user.id}",
            channel=ctx.channel,
            initial=now,
            recur=None,
        )

        unmute_task = Task(
            uid=f"unmute-{ctx.message.id}",
            nicename=f"unmute-{ctx.message.id}",
            author=ctx.author,
            content=f"unmute server {user.id} Scheduler: Scheduled Unmute - {reason}",
            channel=ctx.channel,
            initial=unmute_time,
            recur=None,
        )

        unmute_task_2 = Task(
            uid=f"unmuterole-{ctx.message.id}",
            nicename=f"unmuterole-{ctx.message.id}",
            author=ctx.author,
            content=f"removerole nadeko-mute {user.id}",
            channel=ctx.channel,
            initial=unmute_time,
            recur=None,
        )

        async with self._iter_lock:
            self.scheduled[mute_task.uid] = asyncio.create_task(
                self.delayed_wrap_and_invoke(mute_task, 0)
            )

            self.scheduled[mute_task_2.uid] = asyncio.create_task(
                self.delayed_wrap_and_invoke(mute_task_2, 0)
            )

            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(unmute_task.to_config())
            self.tasks.append(unmute_task)

            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(unmute_task_2.to_config())
            self.tasks.append(unmute_task_2)


# To run this, pay attention to this:
# define num_classes when initializing the model
# define f2c when calling train() and test()

'''Train CIFAR100 with PyTorch.'''
from __future__ import print_function

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms
import dataset

import os
import argparse

from models import *
from utils import progress_bar, adjust_optimizer, setup_logging
from torch.autograd import Variable
from datetime import datetime
import logging
import numpy as np
import pickle

NUM_CLASSES = 2
NUM_CLUSTERS = 5

parser = argparse.ArgumentParser(description='PyTorch CIFAR100 Training')
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
parser.add_argument('--results_dir', metavar='RESULTS_DIR', default='./results',
                    help='results dir')
parser.add_argument('--resume_dir', default=None, help='resume dir')
args = parser.parse_args()

args.save = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
    os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'))
logging.info("saving to %s", save_path)
logging.info("run arguments: %s", args)

use_cuda = torch.cuda.is_available()
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch

# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)),
])

trainset = dataset.data_cifar100_red.CIFAR100_RED(root='/home/rzding/DATA', train=True, download=True, transform=transform_train)
#trainset = dataset.data_cifar10.CIFAR10(root='/home/rzding/DATA', train=True, download=True, transform=transform_train)
#trainloader = torch.utils.data.DataLoader(trainset, batch_size=256, shuffle=True, num_workers=2)

testset = dataset.data_cifar100_red.CIFAR100_RED(root='/home/rzding/DATA', train=False, download=True, transform=transform_test)
#testset = dataset.data_cifar10.CIFAR10(root='/home/rzding/DATA', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=500, shuffle=False, num_workers=2)

# Model
if args.resume:
    # Load checkpoint.
    print('==> Resuming from checkpoint..')
    assert os.path.isdir(args.resume_dir)
    checkpoint = torch.load(os.path.join(args.resume_dir, 'ckpt.t7'))
    net = checkpoint['net']
    best_acc = checkpoint['acc']
    start_epoch = checkpoint['epoch']
else:
    print('==> Building model..')
    # net = VGG('VGG19')
    # net = ResNet18()
    #net = PreActResNet18(num_classes=100)
    net = wide_resnet(num_classes=100)
    # net = GoogLeNet()
    # net = DenseNet121()
    # net = ResNeXt29_2x64d()
    # net = MobileNet()
    # net = MobileNetV2()
    # net = DPN92()
    # net = ShuffleNetG2()
    # net = SENet18()

logging.info("model structure: %s", net)
num_parameters = sum([l.nelement() for l in net.parameters()])
logging.info("number of parameters: %d", num_parameters)

if use_cuda:
    net.cuda()
    net = torch.nn.DataParallel(net, device_ids=[0])
    cudnn.benchmark = True

# Step1: start from a pre-trained model, load it and save the output of last layer
# result format: a matrix where each row is a datapoint, a vector as class of each datapoint
def get_feat(net, trainloader):
    net.eval()
    all_feats = []
    all_idx = []
    all_targets = []
    for batch_idx, (inputs, input_idx, targets) in enumerate(trainloader):
        all_idx.append(input_idx.numpy())
        all_targets.append(targets.numpy())
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        inputs, targets = Variable(inputs, volatile=True), Variable(targets)
        outputs, feats = net(inputs)
        all_feats.append(feats.data.cpu().numpy())
    all_feats = np.vstack(all_feats)
    all_idx = np.hstack(all_idx)
    all_targets = np.hstack(all_targets)
    return all_feats, all_idx, all_targets

trainloader = torch.utils.data.DataLoader(trainset, batch_size=256, shuffle=False, num_workers=2)
train_feats, train_idx, all_targets = get_feat(net, testloader)
pickle.dump(train_feats, open(os.path.join(save_path, 'test_feats.pkl'), 'wb'))
print('all feats size: {}'.format(train_feats.shape))
print('feats sum: {}'.format(train_feats.sum(axis=1)))
print('feats first row: {}'.format(train_feats[0]))
#pickle.dump([train_idx, all_targets], open(os.path.join(save_path, 'debug.pkl'), 'wb'))
pickle.dump([None, all_targets], open(os.path.join(save_path, 'debug.pkl'), 'wb'))

# criterion = nn.CrossEntropyLoss()
# optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
# regime = {
#     0: {'optimizer': 'SGD', 'lr': 1e-1,
#         'weight_decay': 5e-4, 'momentum': 0.9, 'nesterov': False},
#     60: {'lr': 2e-2},
#     120: {'lr': 4e-3},
#     160: {'lr': 8e-4}
# }
# logging.info('training regime: %s', regime)

# # Training
# def train(epoch, f2c=False):
#     print('\nEpoch: %d' % epoch)
#     net.train()
#     train_loss = 0
#     correct = 0
#     total = 0
#     global optimizer
#     optimizer = adjust_optimizer(optimizer, epoch, regime)
#     for batch_idx, (inputs, targets) in enumerate(trainloader):
#         if f2c:
#             for idx,target in enumerate(targets):
#                 targets[idx] = classes_f2c[target]
#         if use_cuda:
#             inputs, targets = inputs.cuda(), targets.cuda()
#         optimizer.zero_grad()
#         inputs, targets = Variable(inputs), Variable(targets)
#         outputs = net(inputs)
#         loss = criterion(outputs, targets)
#         loss.backward()
#         optimizer.step()

#         train_loss += loss.data[0]
#         _, predicted = torch.max(outputs.data, 1)
#         total += targets.size(0)
#         correct += predicted.eq(targets.data).cpu().sum()

#         #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
#         #    % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
#         if batch_idx % 10 == 0:
#             logging.info('\n Epoch: [{0}][{1}/{2}]\t'
#                         'Training Loss {train_loss:.3f} \t'
#                         'Training Prec@1 {train_prec1:.3f} \t'
#                         .format(epoch, batch_idx, len(trainloader),
#                         train_loss=train_loss/(batch_idx+1),
#                         train_prec1=100.*correct/total))

# def test(epoch, f2c=False, train_f=True):
#     global best_acc
#     net.eval()
#     test_loss = 0
#     correct = 0
#     total = 0
#     for batch_idx, (inputs, targets) in enumerate(testloader):
#         if f2c:
#             for idx,target in enumerate(targets):
#                 targets[idx] = classes_f2c[target]
#         if use_cuda:
#             inputs, targets = inputs.cuda(), targets.cuda()
#         inputs, targets = Variable(inputs, volatile=True), Variable(targets)
#         outputs = net(inputs)
#         loss = criterion(outputs, targets)

#         test_loss += loss.data[0]
#         _, predicted = torch.max(outputs.data, 1)
#         total += targets.size(0)
#         if train_f and f2c:
#             predicted_np = predicted.cpu().numpy()
#             #print('predicted: {}'.format(predicted))
#             #print('predicted_np: {}'.format(predicted_np))
#             for idx,a_predicted in enumerate(predicted_np):
#                 predicted_np[idx] = classes_f2c[a_predicted]
#             #correct += (predicted_np == targets.cpu().numpy()).sum()
#             #print('targets: {}'.format(targets))
#             correct += (predicted_np == targets.data.cpu().numpy()).sum()
#         else:
#             correct += predicted.eq(targets.data).cpu().sum()

#         #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
#         #    % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

#     logging.info('\n Epoch: {0}\t'
#                     'Testing Loss {test_loss:.3f} \t'
#                     'Testing Prec@1 {test_prec1:.3f} \t\n'
#                     .format(epoch,
#                     test_loss=test_loss/len(testloader),
#                     test_prec1=100.*correct/total))

#     # Save checkpoint.
#     acc = 100.*correct/total
#     if acc > best_acc and args.resume_dir is None and not (train_f and f2c):
#         print('Saving..')
#         state = {
#             'net': net.module if use_cuda else net,
#             'acc': acc,
#             'epoch': epoch,
#         }
#         torch.save(state, os.path.join(save_path, 'ckpt.t7'))
#         best_acc = acc

# for epoch in range(start_epoch, start_epoch+200):
#     train(epoch, f2c=False)
#     test(epoch, f2c=False)
#     test(epoch, f2c=True, train_f=True)

# # test(0, f2c=True, train_f=True)


#!/usr/bin/env python
# this script extracts gene/transcript ID and TE family info from an insertions/excisions results file and outputs a rearranged file
# USE: process_gene_interrupt.py <all_insertions_and_excisions_file>
import sys
import re

in_file = sys.argv[1]
IN_FILE = open(in_file, "r")

out_file = "final_{in_file}".format(**locals())
OUT_FILE = open(out_file, "w")
for line in IN_FILE:
	line = line.rstrip('\n')
	items = re.split("[\t]", line)
	chromosome = items[2]
	category = items[3]
	start_pos = items[5]
	te_start = items[8]
	te_info = items[9]
	te_orient = items[11]
	match = re.search("(.*)_(\w+-)?reference", te_info)
	family = match.group(1)

	ID_info = items[7]
	match2 = re.search("(?:Pseudogene|Transcript|sequence_name|^Name)(?:=|:)([\w|\d]+.\d+)", ID_info) #jsut pull gene name, remove splice info
	ID = match2.group(1)

	other_fields = '\t'.join(items[0:7])

	OUT_FILE.write("{other_fields}\t{ID}\t{te_start}\t{family}\t{te_orient}\n".format(**locals()))

IN_FILE.close()


#!/usr/bin/python3
"""
    This file contains class definitions of a node
    and a  singly linked list
"""

class Node:
    """Node of a singly linked list.
    Private instance attribute: data:
        - property def data(self)
        - property setter def data(self, value)
    Private instance attribute: next_node:
        - property def next_node(self)
        - property setter def next_node(self, value)
    Instantiation with data and next_node.
    """

    def __init__(self, data, next_node=None):
        self.data = data
        self.next_node = next_node

    @property
    def data(self):
        return self.__data

    @property
    def next_node(self):
        return self.__next_node

    @data.setter
    def data(self, value):
        if type(value) is not int:
            raise TypeError("data must be an integer")
        self.__data = value

    @next_node.setter
    def next_node(self, value):
        if value is None or isinstance(value, Node):
            self.__next_node = value
        else:
            raise TypeError("next_node must be a Node object")

class SinglyLinkedList:
    """ Singly linked list.
    Private instance attribute: head.
    Simple instantiation.
    Public instance method: def sorted_insert(self, value).
    """

    def __init__(self):
        self.__head = None

    def sorted_insert(self, value):
        if self.__head is None:
            self.__head = Node(value)
            return
        node = self.__head
        prev = None
        while (node and node.data < value):
            prev = node
            node = node.next_node
        if node == self.__head:
            new_node = Node(value)
            new_node.next_node = self.__head
            self.__head = new_node
            return
        new_node = Node(value)
        new_node.next_node = node
        prev.next_node = new_node
        return

    def __str__(self):
        node = self.__head
        _str = ""
        while (node):
            _str += str(node.data)
            if node.next_node is not None:
                _str += "\n"
            node = node.next_node
        return _str


from sklearn.ensemble import RandomForestRegressor
import xgboost as xgb
import pandas as pd

def recover_missing(df, input_attr, missing_attr, method='RandomForest'):
    """
    :param df: input dataframe
    :param input_attr: type -> list:
    :param missing_attr: str: attrb need to be recover
    :param method: which method choose to recover data -> 'RandomForest', 'Xgboost', 'lightgbm'
    :return: df
    """
    attrb_df = df[[missing_attr] + input_attr]
    train = attrb_df[getattr(attrb_df, missing_attr).notnull()].values
    test = attrb_df[getattr(attrb_df, missing_attr).isnull()].values

    train_label = train[:, 0]
    train_data = train[:, 1:]

    if method == 'RandomForest':
        model = RandomForestRegressor(random_state=1, n_estimators=2000, n_jobs=-1)
        model.fit(train_data, train_label)
        predict_result = model.predict(test[:, 1:])
        df.loc[(hasattr(df, missing_attr).isnull()), missing_attr] = predict_result
    elif method == 'Xgboost':
        best_params = {
            'max_depth': 5,
            'learning_rate': 0.01,
            'n_estimators': 3000,
            'gamma': 0.8,
            'min_child_weight': 2,
            'reg_alpha': 0.001,
            'max_delta_step': 0,
            'subsample': 0.8,
            'colsample_bytree': 0.8,
            'colsample_bylevel': 0.9,
            'base_score': 10,
            'seed': 1,
            'nthread': 30
        }
        model = xgb.XGBRegressor(**best_params)
        model.fit(train_data, train_label)
        predict_result = model.predict(test[:, 1:])
        df.loc[(getattr(df, missing_attr).isnull()), missing_attr] = predict_result
    return df, model

def feature_transform(df, func):
    nrow, ncol = df.shape
    for i in range(nrow):
        func(df, i)
    return df

if __name__ == '__main__':
    data = pd.read_csv('../../data/titanic_disaster/train.csv')
    df = recover_missing(data, ['Fare', 'Parch', 'SibSp', 'Pclass'], 'Age', method='Xgboost')