import logging
import boto3
logger = logging.getLogger()
logger.setLevel(logging.INFO)
ec2 = boto3.resource('ec2')

#Change value based on ISO 3166-1 alpha-2 country code
COUNTRY_CODE = 'es'
STOPPED_INSTANCE_CODE = 80

def is_stopped(instance):
    if instance.state['Code'] == STOPPED_INSTANCE_CODE:
        return True
    return False

def has_country(country, instance):
    tags = instance.tags
    for tag in tags:
        if (tag['Key'] == 'country') & (tag['Value'] == country):
            return True
    return False

def get_stopped_instances_id(country):
    stop_instances_id = []
    instances = list(ec2.instances.all())
    logger.info("Number of instances found: {}".format(len(instances)))
    for instance in instances:
        if has_country(country,instance) & is_stopped(instance):
            stop_instances_id.append(instance.instance_id)
    return stop_instances_id

def start_instances(country):
    stopped__instances_id = get_stopped_instances_id(country)
    logger.info("{} number of instances that will be start".format(str(len(stopped__instances_id))))
    for id in stopped__instances_id:
        ec2.Instance(id).start()
        logger.info('Instance with id {} has been started'.format(id)  )

def lambda_handler(event, context):
    country=COUNTRY_CODE
    logger.info('Starting instances of: {}'.format(country))
    start_instances(country)


# coding: utf-8

# In[6]:

from tensorflow.examples.tutorials.mnist import input_data

# In[7]:

mnist = input_data.read_data_sets("data_set/MNIST_data",one_hot=True)

# In[8]:

print('训练数据：',mnist.train.images)
print('训练数据打印shape：',mnist.train.images.shape)

# In[9]:

import pylab #图形打印类

# In[12]:

im=mnist.train.images[1]
im=im.reshape(-1,28) #将图由一行784个像素，转换成28*28像素的图像
pylab.imshow(im)
pylab.show()

# In[14]:

print('测试数据打印shape',mnist.test.images.shape)
print('验证数据打印shape',mnist.validation.images.shape)

# ### mnist 分类

# In[15]:

from tensorflow.examples.tutorials.mnist import input_data
mnist=input_data.read_data_sets('第03周/MNIST_data',one_hot=True)

# In[16]:

tf.reset_default_graph()

# #### 搭建模型

# In[17]:

X=tf.placeholder(tf.float32,[None,784])
Y=tf.placeholder(tf.float32,[None,10])

# In[18]:

W=tf.Variable(tf.random_normal([784,10]))
b=tf.Variable(tf.zeros([10]))

# ### 预测的是属于某一个类别的概论，总和为1 ，例如[0.2,0.6,0.2]

# In[19]:

pred=tf.nn.softmax(tf.matmul(X,W) + b)  # softmax 分类

# ### 反向传播，将生成的pred与样本标签Y进行一次交叉熵运算最小化误差cost

# In[22]:

cost=tf.reduce_mean(-tf.reduce_sum(Y*tf.log(pred),reduction_indices=1))
#cost=tf.nn.softmax_cross_entropy_with_logits(tf.matmul(X,W) + b)

# In[24]:

learning_rate=0.01
optimizer=tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

# In[26]:

training_epochs=20
batch_size=100
display_step=1
saver=tf.train.Saver()
model_path='log/mnist_model.ckpt'

# In[31]:

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    for epoch in range(training_epochs):
        avg_cost=0
        total_batch=int(mnist.train.num_examples/batch_size) # 每一轮训练多少批次
        #遍历全部数据
        for i in range(total_batch):
            batch_xs,batch_ys=mnist.train.next_batch(batch_size)
            _,c=sess.run([optimizer,cost],feed_dict={X:batch_xs,Y:batch_ys})

            #计算平均值以使误差值更加平均
            avg_cost += c / total_batch

        #显示训练中详细信息
        if (epoch+1) % display_step == 0:
            print('Epoch:','%04d'%(epoch+1),'cost=','{:.9f}'.format(avg_cost))
    print('Finished')

    #测试模型
    correct_pred=tf.equal(tf.argmax(pred,1),tf.argmax(Y,1))
    #计算准确率
    accuracy=tf.reduce_mean(tf.cast(correct_pred,tf.float32)) #cast函数，将correct_pred数据类型转换成float32

    #print('Accuracy:',accuracy.eval({X:mnist.test.images,Y:mnist.test.labels}))
    print('Accuracy:',sess.run(accuracy,feed_dict={X:mnist.test.images,Y:mnist.test.labels}))

    #存储模型
    model=saver.save(sess,model_path)
    print('Model saved in file: %s'%model)

# ### 读取模型，使用模型

# In[38]:

saver=tf.train.Saver()
model_path='log/mnist_model.ckpt'
with tf.Session() as sess2:
    tf.global_variables_initializer().run()
    saver.restore(sess2,model_path)
    print(saver)

    correct_prediction=tf.equal(tf.argmax(pred,1),tf.argmax(Y,1))
    accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
    print('Accuracy:',accuracy.eval({X:mnist.test.images,Y:mnist.test.labels}))

    output=tf.argmax(pred,1)

    #获取两个手写数字，并使用模型进行预测
    batch_xs,batch_ys=mnist.train.next_batch(2)
    outputval,predv=sess2.run([output,pred],feed_dict={X:batch_xs})
    print(outputval)
    print(predv)
    print(batch_ys)


import jieba

jieba.load_userdict("TsinghuaDict.txt")

class wordsCut:
    #创建停用词表
    @classmethod
    def stop_words_list(cls,filepath):
        with open(filepath,'r',encoding='utf-8') as file:
            stop_words = [line.strip() for line in file.readlines()]
            return stop_words

    #对搜索语句进行中文分词
    @classmethod
    def sentence_seg(cls,query):
        query_seg = jieba.cut(query.strip())
        stop_words = cls.stop_words_list('stop_words.txt')
        keywords = []
        for word in query_seg:
            if word not in stop_words and word != '\t':
                keywords.append(word)
        return keywords


def Min(a, b):
    if a < b:
        return a
    else:
        return b

def Max(x, y):
    if x > y:
        return x
    else:
        return y

a, b = map(float, input().split())

u = Min(a, b)
v = Min(a * b, Max(a, b))
s = Min(u + v, 3.14)

print('%.2f %.2f %.2f' % (u, v, s))


import mdlpoints as mdlp # utilizes the dictionary for maximum dead lift points
import sptpoints as sptp # utilizes the dictionary for standing power throw points
import hrppoints as hrpp # utilizes the dictionary for hand release pushup points
import sdcpoints as sdcp # utilizes the dictionary for sprint drag carry points
import plnkpoints as plnkp # utilizes the dictionary for the plank points
import tmrpoints as tmrp # utilizes the dictionary for the two mile run points
from flask import Flask, request, jsonify, render_template

app = Flask(__name__)

event_dicts = {
    'male': {
        '17-21': {
            'mdl': mdlp.mdl1721m_dict,
            'hrp': hrpp.hrp1721m_dict,
            'spt': sptp.spt1721m_dict,
            'sdc': sdcp.sdc1721m_dict,
            'plnk': plnkp.plnk1721_dict,
            'tmr': tmrp.tmr1721m_dict
        },
        '22-26': {
            'mdl': mdlp.mdl2226m_dict,
            'hrp': hrpp.hrp2226m_dict,
            'spt': sptp.spt2226m_dict,
            'sdc': sdcp.sdc2226m_dict,
            'plnk': plnkp.plnk2226_dict,
            'tmr': tmrp.tmr2226m_dict
        },
        '27-31': {
            'mdl': mdlp.mdl2731m_dict,
            'hrp': hrpp.hrp2731m_dict,
            'spt': sptp.spt2731m_dict,
            'sdc': sdcp.sdc2731m_dict,
            'plnk': plnkp.plnk2731_dict,
            'tmr': tmrp.tmr2731m_dict
        },
        '32-36': {
            'mdl': mdlp.mdl3236m_dict,
            'hrp': hrpp.hrp3236m_dict,
            'spt': sptp.spt3236m_dict,
            'sdc': sdcp.sdc3236m_dict,
            'plnk': plnkp.plnk3236_dict,
            'tmr': tmrp.tmr3236m_dict
        },
        '37-41': {
            'mdl': mdlp.mdl3741m_dict,
            'hrp': hrpp.hrp3741m_dict,
            'spt': sptp.spt3741m_dict,
            'sdc': sdcp.sdc3741m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr3741m_dict
        },
        '42-46': {
            'mdl': mdlp.mdl4246m_dict,
            'hrp': hrpp.hrp4246m_dict,
            'spt': sptp.spt4246m_dict,
            'sdc': sdcp.sdc4246m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr4246m_dict
        },
        '47-51': {
            'mdl': mdlp.mdl4751m_dict,
            'hrp': hrpp.hrp4751m_dict,
            'spt': sptp.spt4751m_dict,
            'sdc': sdcp.sdc4751m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr4751m_dict
        },
        '52-56': {
            'mdl': mdlp.mdl5256m_dict,
            'hrp': hrpp.hrp5256m_dict,
            'spt': sptp.spt5256m_dict,
            'sdc': sdcp.sdc5256m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr5256m_dict
        },
        '57-61': {
            'mdl': mdlp.mdl5761m_dict,
            'hrp': hrpp.hrp5761m_dict,
            'spt': sptp.spt5761m_dict,
            'sdc': sdcp.sdc5761m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr5761m_dict
        },
        '62+': {
            'mdl': mdlp.mdl62m_dict,
            'hrp': hrpp.hrp62m_dict,
            'spt': sptp.spt62m_dict,
            'sdc': sdcp.sdc62m_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr62m_dict
        },
    },
    'female': {
        '17-21': {
            'mdl': mdlp.mdl1721f_dict,
            'hrp': hrpp.hrp1721f_dict,
            'spt': sptp.spt1721f_dict,
            'sdc': sdcp.sdc1721f_dict,
            'plnk': plnkp.plnk1721_dict,
            'tmr': tmrp.tmr1721f_dict
        },
        '22-26': {
            'mdl': mdlp.mdl2226f_dict,
            'hrp': hrpp.hrp2226f_dict,
            'spt': sptp.spt2226f_dict,
            'sdc': sdcp.sdc2226f_dict,
            'plnk': plnkp.plnk2226_dict,
            'tmr': tmrp.tmr2226f_dict
        },
        '27-31': {
            'mdl': mdlp.mdl2731f_dict,
            'hrp': hrpp.hrp2731f_dict,
            'spt': sptp.spt2731f_dict,
            'sdc': sdcp.sdc2731f_dict,
            'plnk': plnkp.plnk2731_dict,
            'tmr': tmrp.tmr2731f_dict
        },
        '32-36': {
            'mdl': mdlp.mdl3236f_dict,
            'hrp': hrpp.hrp3236f_dict,
            'spt': sptp.spt3236f_dict,
            'sdc': sdcp.sdc3236f_dict,
            'plnk': plnkp.plnk3236_dict,
            'tmr': tmrp.tmr3236f_dict
        },
        '37-41': {
            'mdl': mdlp.mdl3741f_dict,
            'hrp': hrpp.hrp3741f_dict,
            'spt': sptp.spt3741f_dict,
            'sdc': sdcp.sdc3741f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr3741f_dict
        },
        '42-46': {
            'mdl': mdlp.mdl4246f_dict,
            'hrp': hrpp.hrp4246f_dict,
            'spt': sptp.spt4246f_dict,
            'sdc': sdcp.sdc4246f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr4246f_dict
        },
        '47-51': {
            'mdl': mdlp.mdl4751f_dict,
            'hrp': hrpp.hrp4751f_dict,
            'spt': sptp.spt4751f_dict,
            'sdc': sdcp.sdc4751f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr4751f_dict
        },
        '52-56': {
            'mdl': mdlp.mdl5256f_dict,
            'hrp': hrpp.hrp5256f_dict,
            'spt': sptp.spt5256f_dict,
            'sdc': sdcp.sdc5256f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr5256f_dict
        },
        '57-61': {
            'mdl': mdlp.mdl5761f_dict,
            'hrp': hrpp.hrp5761f_dict,
            'spt': sptp.spt5761f_dict,
            'sdc': sdcp.sdc5761f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr5761f_dict
        },
        '62+': {
            'mdl': mdlp.mdl62f_dict,
            'hrp': hrpp.hrp62f_dict,
            'spt': sptp.spt62f_dict,
            'sdc': sdcp.sdc62f_dict,
            'plnk': plnkp.plnk3762_dict,
            'tmr': tmrp.tmr62f_dict
        }
    }
}

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/calculate_acft', methods=['POST'])
def handle_calculate_acft():
    try:
        data = request.get_json()
        acft_score = calculate_acft_score(data)
        response_data = {'acft_score': acft_score}
        return jsonify(response_data)
    except Exception as e:
        return jsonify({'error': 'An error occurred while calculating the ACFT score: ' + str(e)}), 400

def calculate_acft_score(data):
    age = data['age']
    gender = data['gender']
    mdl = int(data['mdl'])
    hrp = int(data['hrp'])
    spt = float(data['spt'])
    sdc_input = data['sdc']
    plnk_input = data['plnk']
    tmr_input = data['tmr']
#Print for troubleshooting
    print("Received gender:", gender)
    print("Received age:", age)
    print("Received mdl:", mdl)
    print("Received hrp:", hrp)
    print("Received spt:", spt)
    print("Received sdc:", sdc_input)
    print("Received plnk:", plnk_input)
    print("Received tmr:", tmr_input)

    event_scores = event_dicts.get(gender, {}).get(age, {})
    mdl_points = event_scores.get('mdl', {}).get(mdl, 0)
    hrp_points = event_scores.get('hrp', {}).get(hrp, 0)
    spt_points = event_scores.get('spt', {}).get(spt, 0)
    sdc_points = event_scores.get('sdc', {}).get(sdc_input, 0)
    plnk_points = event_scores.get('plnk', {}).get(plnk_input, 0)
    tmr_points = event_scores.get('tmr', {}).get(tmr_input, 0)

    if None in [mdl_points, hrp_points, spt_points, sdc_points, plnk_points, tmr_points]:
        return 'Invalid input for one or more events'

    acft_score = mdl_points + hrp_points + spt_points + sdc_points + plnk_points + tmr_points

    if isinstance(acft_score, int):
        acft_score = int(acft_score)

    return acft_score

if __name__ == '__main__':
    app.run(debug=True)


from flask import render_template, request, session, make_response, redirect, url_for
from . import app, db
from .models import Subject, Trial
import pandas as pd
import random
import numpy as np
from sqlalchemy import or_

subs_per_p2 = 2
ntrials = 3

@app.route('/')
def index():
    # on remote set these
    # prolific_id = request.args.get('PROLIFIC_PID'),
    # flask_subId = request.args.get('flasker')

    return redirect(url_for('welcome', PROLIFIC_PID=np.random.random(),
                            SESSION_ID=np.random.random(), trial=0))

@app.route('/welcome', methods=['GET', 'POST'])
def welcome():
    return render_template('welcome.html')

@app.route('/consent', methods=['GET', 'POST'])
def consent():
    if request.method == 'GET':
        return render_template('consent.html')
    if request.method == 'POST':
        return make_response("200")

@app.route('/new_subject')
def new_subject():
    # query DB to find out what trustee/strategy subject will watch
    subjs = np.array([subj.trustee_id for subj in Subject.query.filter_by(complete=True).all()])
    # db.session.query(Subject).filter(or_(Subject.complete == True, Subject.in_progress == True)).all()])
    for p2 in [93, 97, 54, 62]:
        if len(np.where(subjs == p2)[0]) >= subs_per_p2:
            continue
        else:
            trustee_to_observe = p2
            break
    # Populated Database for New Subject
    # Probes: fixed after first trial, then uniformly sampled
    probes = [1]
    [probes.append(np.random.choice([t - 1, t, t + 1])) for t in range(1, ntrials) if t % 4 == 0]
    # Local
    game_dat = pd.read_csv('possibility_app/static/stim_data/HMTG_possib_stim.csv', header=0, index_col=0)
    # Remote
    # game_dat = pd.read_csv('/home/bryan/HMTG_possibilities/possibility_app/static/stim_data/HMTG_possib_stim.csv', header=0, index_col=0)
    game_dat = game_dat.loc[game_dat.trustee == trustee_to_observe]
    game_dat = game_dat.iloc[:ntrials]  # beta test, less trials
    game_dat = game_dat.sample(frac=1, random_state=np.random.RandomState()).reset_index(drop=True)
    game_dat['trial'] = range(1, ntrials + 1)
    # add subject
    practice = []
    practice.insert(0, {'trustee': trustee_to_observe, 'trial': 0, 'inv': 10, 'mult': 4, 'ret': 15, 'im': 40,
                        'trustee_clust': game_dat.trustee_clust.values[0]})
    game_dat = pd.concat([pd.DataFrame(practice), game_dat], ignore_index=True)
    subj = Subject(prolific_id=request.args.get('PROLIFIC_PID'),
                   in_progress=True,
                   complete=False,
                   session_id=request.args.get('SESSION_ID'),
                   trustee_id=trustee_to_observe,
                   trustee_strategy=game_dat.trustee_clust.values[0])

    db.session.add(subj)
    # add trials
    for t in range(len(game_dat)):
        db.session.add(
            Trial(trl=t,
                  p1_pic=np.random.randint(int(79)),
                  inv=int(game_dat['inv'][t]),
                  mult=int(game_dat['mult'][t]),
                  ret=int(game_dat['ret'][t]),
                  probe=t in probes,
                  prolific_id=subj.prolific_id)
        )
    db.session.commit()
    print('You have a new Subject', subj.prolific_id)
    return redirect(url_for('instructions', PROLIFIC_PID=request.args.get('PROLIFIC_PID'), SESSION_ID=request.args.get('SESSION_ID'),
                            trial=int(request.args.get('trial'))))

@app.route('/instructions', methods=['GET', 'POST'])
def instructions():
    if request.method == 'GET':

        return render_template('instructions.html')
    else:
        return make_response("200")

@app.route('/practice', methods=['GET', 'POST'])
def practice():
    if request.method == 'GET':
        tdat = Trial.query.filter_by(prolific_id=request.args.get('PROLIFIC_PID'), trl=int(request.args.get('trial'))).first()
        return render_template('invest.html', trial_num=tdat.trl,
                               p1=tdat.p1_pic,
                               inv_amt=tdat.inv,
                               mult=tdat.mult,
                               ntrials=ntrials)

@app.route('/ready')
def ready():
    return render_template('ready.html')

@app.route('/next_trial')
def next_trial():
    return redirect(url_for('invest', PROLIFIC_PID=request.args.get('PROLIFIC_PID'), SESSION_ID=request.args.get('SESSION_ID'),
                            trial=int(int(request.args.get('trial')) + 1)))

@app.route('/invest', methods=['GET', 'POST'])
def invest():
    if request.method == 'GET':
        tdat = Trial.query.filter_by(prolific_id=request.args.get('PROLIFIC_PID'), trl=int(request.args.get('trial'))).first()
        return render_template('invest.html', trial_num=tdat.trl,
                               p1=tdat.p1_pic,
                               inv_amt=tdat.inv,
                               mult=tdat.mult,
                               ntrials=ntrials)

@app.route('/predict', methods=['GET', 'POST'])
def predict():
    if request.method == 'GET':
        tdat = Trial.query.filter_by(prolific_id=request.args.get('PROLIFIC_PID'),
                                     trl=int(request.args.get('trial'))).first()
        return render_template('predict.html', trial_num=tdat.trl,
                               p1=tdat.p1_pic,
                               inv_amt=tdat.inv,
                               mult=tdat.mult,
                               ntrials=ntrials)

    elif request.method == 'POST':
        trial_dat = request.get_json()
        tdat = Trial.query.filter_by(prolific_id=trial_dat['PROLIFIC_PID'], trl=int(trial_dat['trial'])).first()
        tdat.pred = int(trial_dat['trial_prediction'])
        tdat.guess_fill = int(trial_dat['guess_fill'])
        db.session.add(tdat)
        db.session.commit()
        print(f'subject: {tdat.prolific_id},trial:{trial_dat["trial"]} , predicted: {trial_dat["trial_prediction"]}')
        return make_response("200")

@app.route('/decision', methods=['GET', 'POST'])
def decision():
    tdat = Trial.query.filter_by(prolific_id=request.args.get('PROLIFIC_PID'), trl=int(request.args.get('trial'))).first()
    if tdat.probe:
        interval = 111
    else:
        interval = 999
    return render_template('decision.html',
                           trial_num=tdat.trl,
                           p1=tdat.p1_pic,
                           inv_amt=tdat.inv,
                           mult=tdat.mult,
                           pred_amt=tdat.pred,
                           guess_fill=tdat.guess_fill,
                           ret=tdat.ret,
                           interval=interval,
                           last_trl=ntrials,
                           ntrials=ntrials)

@app.route('/guesswhy', methods=['GET', 'POST'])
def guessWhy():
    if request.method == 'GET':
        tdat = Trial.query.filter_by(prolific_id=request.args.get('PROLIFIC_PID'),
                                     trl=int(request.args.get('trial'))).first()
        return render_template('guesswhy.html',
                               trial_num=tdat.trl,
                               p1=tdat.p1_pic,
                               inv_amt=tdat.inv,
                               mult=tdat.mult,
                               ret=tdat.ret,
                               pred_amt=tdat.pred,
                               guess_fill=tdat.guess_fill,
                               ntrials=ntrials)

    elif request.method == 'POST':
        answer = request.get_json()
        tdat = Trial.query.filter_by(prolific_id=answer['PROLIFIC_PID'],
                                     trl=int(answer['trial'])).first()

        tdat.reason = answer['subject_response']
        tdat.reason_start = answer['resp_start']
        tdat.reason_rt = answer['resp_end']
        db.session.add(tdat)
        db.session.commit()
        print(f'subject: {tdat.prolific_id},trial:{answer["trial"]} , responded: {answer["subject_response"]}')
        return make_response("200")

@app.route('/thanks', methods=['GET', 'POST'])
def thanks():
    if request.method == 'GET':
        pp_id = request.args.get('PROLIFIC_PID')
        subj = Subject.query.filter_by(prolific_id=pp_id).first()
        tt = random.randint(1, ntrials-1)
        trl = Trial.query.filter_by(trl=tt, prolific_id=pp_id).first()
        pe = abs((trl.pred/(trl.inv*trl.mult)) - (trl.ret/(trl.inv*trl.mult)))
        acc = 1 - pe
        bonus = round((2 * acc), 2)
        subj.bonus = bonus
        db.session.add(subj)
        db.session.commit()
        return render_template('thanks.html', bonus=bonus)

    elif request.method == 'POST':
        answer = request.get_json()
        subj = Subject.query.filter_by(prolific_id=answer['PROLIFIC_PID']).first()

        subj.exp_feedback = answer['subject_feedback']
        subj.complete = True
        subj.in_progress = False
        db.session.add(subj)
        db.session.commit()
        print(f"Subject {subj.prolific_id} has finished!!!")
        return make_response('200')


# circle_graph_filtered_file_prep.py

import os
import sys
import json
from utils import *

USAGE_STR = """
# Purpose
# Takes an dynamic interaction dictionary and converts to a comma delimited file format
# that can be read by the d3 circle graph animator.

# Coloring
# Green - Direct hydrogen bond
# Blue - Water Bridge
# Red - Extended Water Bridge

# Usage
# python bundle_plot_prep.py <INTERACTION_DICT_FOLDER> <PDB_CDOE> <OUTPUT_FILE> <FILTER_LIST>

# Arguments
# <INTERACTION_DICT> Absolute path to the dictionary containing key = interaction and
# value = time points that the interaction is present in trajectory
# <PDB_CDOE> PDB of the receptor associated with this interaction dictionary
# <FILTER_LIST> Absolute path to the file to filter the interactions
# <OUTPUT_FILE> File of output path to store the newly formatted file

# Example
# INTERACTION_DICT="/scratch/PI/rondror/akma327/DynamicNetworks/data/DynamicNetworksOutput/InteractionOutputDictionary/DOR-inactive-naltrindole-unpublished/condition-naltrindole-bound/rep_1/050516_NONHB_DEFAULT_HB_3.5_70"
# PDB_CODE="4N6H"
# FILTER_LIST="/scratch/PI/rondror/akma327/DynamicNetworks/data/crystal-analysis/interaction-footprint-analysis/polar-network-lists/inactive-dor-network.txt"
# OUTPUT_FILE="/scratch/PI/rondror/akma327/DynamicNetworks/data/crystal-analysis/interaction-footprint-analysis/circle-graph-input/dor-inactive-filtered.json"
# python bundle_plot_prep.py $INTERACTION_DICT $PDB_CODE $FILTER_LIST $OUTPUT_FILE

"""

K_MIN_ARG = 5
MAX_LINE_WIDTH = 12
LINE_COLOR_DICT = {"hbss": "#22AA66", "hbbb": "#22AA66", "hbsb": "#22AA66", "wb": "#2266AA", "wb2": "#AA2222"}

def genTypeResiduePairDict(FILTER_LIST):
	type_residue_pair_dict = {} # Key = Interaction Type, value = List of residue pairs
	POLAR_NETWORK_PDB = None
	f = open(FILTER_LIST, 'r')
	for line in f:
		if("PDB:" in line):
			POLAR_NETWORK_PDB = line.split("PDB:")[1].strip()
		if(" -- " in line):
			linfo = line.split("@")
			respair, interaction_type = linfo[0].strip(), linfo[1].strip()
			aapair = respair.split(" -- ")
			short_aa1, short_aa2 = aapair[0].strip(), aapair[1].strip()
			aa1 = aa1to3_dict[short_aa1[0:1]].upper() + short_aa1[1:]
			aa2 = aa1to3_dict[short_aa2[0:1]].upper() + short_aa2[1:]
			key1, key2 = (aa1, aa2), (aa2, aa1)
			if(interaction_type not in type_residue_pair_dict):
				type_residue_pair_dict[interaction_type] = [key1]
			else:
				if(key1 not in type_residue_pair_dict[interaction_type] and key2 not in type_residue_pair_dict[interaction_type]):
					type_residue_pair_dict[interaction_type].append(key1)
	return POLAR_NETWORK_PDB, type_residue_pair_dict

# Process a single interaction type's dictionary
def reformatSingleFile(json_dict, interaction_type, interaction_dict_path, gpcdbFilterList):
	print(interaction_dict_path)
	f = open(interaction_dict_path, 'r')
	TotalFrames = None
	for line in f:
		if("TotalFrames:" in line):
			TotalFrames = int(line.split("TotalFrames:")[1].strip())
		if(" -- " in line):
			l_info = line.split("~")
			atoms = l_info[0].strip().split(" -- ")
			resatom1 = atoms[0].split("-")
			res1, atom1 = resatom1[0].strip(), resatom1[1].strip()
			resatom2 = atoms[1].split("-")
			res2, atom2 = resatom2[0].strip(), resatom2[1].strip()
			gpcrdb1, gpcrdb2 = getGPCRDB(res1, GPCRDB_DICT), getGPCRDB(res2, GPCRDB_DICT)
			if(gpcrdb1 == "-" or gpcrdb2 == "-"): continue
			if((gpcrdb1, gpcrdb2) not in gpcdbFilterList and (gpcrdb2, gpcrdb1) not in gpcdbFilterList):
				# print(gpcrdb1, gpcrdb2)
				continue
			print(gpcrdb1, gpcrdb2, "match")
			prefix1 = gpcrdb1[0:1]
			prefix2 = gpcrdb2[0:1]

			# Add interaction entry into json_dict
			# Names
			name1 = prefix1 + "." + gpcrdb1 + ":" + atom1
			name2 = prefix2 + "." + gpcrdb2 + ":" + atom2
			# Frequency
			timepoints = l_info[1].strip()
			timepoints = timepoints.strip("[,]")
			frequency = float(timepoints.count(",") + 1)/TotalFrames
			# Line width and color
			width = MAX_LINE_WIDTH*frequency
			color = LINE_COLOR_DICT[interaction_type]
			single_interaction_info_dict = {"name1":name1, "name2": name2, "color": color, "width": width, "frames": [0]}
			json_dict["interactions"].append(single_interaction_info_dict)

def genGpcdbFilterList(aa_filter_list, POLAR_NETWORK_GPCRDB_DICT):
	return [(getGPCRDB(aapair[0], POLAR_NETWORK_GPCRDB_DICT), getGPCRDB(aapair[1], POLAR_NETWORK_GPCRDB_DICT)) for aapair in aa_filter_list]

def reformatAggregateFile(INTERACTION_DICT_FOLDER, type_residue_pair_dict, OUTPUT_FILE, GPCRDB_DICT, POLAR_NETWORK_GPCRDB_DICT):
	json_dict = {"interactions" : []}
	for interaction_type in sorted(type_residue_pair_dict.keys()): # iterate through each interaction type in dictionary folder
		interaction_dict_path = INTERACTION_DICT_FOLDER + "/" + flag_to_dict_file[interaction_type]
		aa_filter_list = type_residue_pair_dict[interaction_type]
		gpcdbFilterList = genGpcdbFilterList(aa_filter_list, POLAR_NETWORK_GPCRDB_DICT)
		print(gpcdbFilterList)
		print("process single dict")
		reformatSingleFile(json_dict, interaction_type, interaction_dict_path, gpcdbFilterList)
	with open(OUTPUT_FILE, 'w') as f:
		json.dump(json_dict, f)

if __name__ == "__main__":
	if(len(sys.argv) < K_MIN_ARG):
		print(USAGE_STR)
		exit(1)
	INTERACTION_DICT_FOLDER = sys.argv[1]
	PDB_CODE = sys.argv[2]
	FILTER_LIST = sys.argv[3]
	OUTPUT_FILE = sys.argv[4]
	createDirectory(OUTPUT_FILE)
	GPCRDB_DICT = genResidueToGpcrdbTable(PDB_CODE) #res to gpcrdb
	POLAR_NETWORK_PDB, type_residue_pair_dict = genTypeResiduePairDict(FILTER_LIST)
	POLAR_NETWORK_GPCRDB_DICT = genResidueToGpcrdbTable(POLAR_NETWORK_PDB)
	# print("Residues from the polar network list")
	# print(type_residue_pair_dict)
	reformatAggregateFile(INTERACTION_DICT_FOLDER, type_residue_pair_dict, OUTPUT_FILE, GPCRDB_DICT, POLAR_NETWORK_GPCRDB_DICT)


# _*_ coding:utf-8 _*_
import requests
import re

url = "http://files2.mca.gov.cn/cws/201502/20150225155630452.htm"
headers = {
    'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
}
response = requests.get(url,headers = headers)

# 匹配所有非空字符
pat1 = r'<td.*>(\S+)<'
# pat1 = r'<td class.*>(\S+)</td>'

# 查询出所有匹配出的字符串结果
mydata  = re.findall(pat1,response.text)

# print(len(mydata))     取出下标的[1]到[6438]的数据

f = open("F:/mydata/年中华人民共和国县以上行政区划代码表.csv", 'w')

# 行政区划代码的数据处理
for i in range(0,len(mydata)-2):
    if ( i == 0 ):
         print ('行政区划代码', ',', '单位名称', file=f)

    if( (i % 2) == 1 and i>=2):
         print (mydata[i], ',', mydata[i+1], file=f)

f.close()


#least common multiple
#parameter should be an array
#I increment by the maximum value as it has to be divisble either ways
def findLCM(x):
    print(x);
    #variable to be divided by
    maximum=max(x)
    LCM = maximum;
    Run = True;

    #setting a not divisible flag
    NotDiv = False;

    ##if one variable is not perfectly divisible then set a flag##
    while(Run == True):
        #check if divisble (set flag if not)
        for i in range(0,len(x)):
            if(LCM%x[i] != 0):
                NotDiv = True;
                break;
        #end while loop if flag not set
        if(NotDiv == False):
            print("LCM =",LCM);
            Run = False;

        #reset flag and increment by an appropriate value
        NotDiv = False;
        LCM= LCM + maximum;
    print("Done")

#testing
var=[3,5,8]
findLCM(var)


import httplib
from uuid import uuid4
from logging import getLogger
from flask import request
from flask import abort as flask_abort
from flask_restful import Resource
from flask_restful.reqparse import RequestParser
from flask_jwt import current_identity
from ..auth.roles import require_user
from ..auth.roles import require_admin
from ..events.models import create_event
from .models import Object
from .models import create_object
from .models import get_object_by_id
from .models import get_type_object
from .models import get_object_child
from .models import get_object_children
from .type_class import get_object_action
from .type_class import get_type_action
from .all_type_classes import all_type_classes
from .labs import Lab

logger = getLogger(__name__)

#----------------------------------------------------------#
# Object lookup                                            #
#----------------------------------------------------------#

def _object_path_parts(obj_path):
    '''Splits the object path `obj_path` into path and action.

    >>> _object_path_parts('a/')
    (['a'], '')
    >>> _object_path_parts('a/b')
    (['a'], 'b')
    >>> _object_path_parts('a/b/')
    (['a', 'b'], '')
    >>> _object_path_parts('a/b/~/xxx')
    (['a', 'b', '~'], 'xxx')
    >>> _object_path_parts('')
    Traceback (most recent call last):
    ...
    NotFound: 404: Not Found
    >>> _object_path_parts('a')
    Traceback (most recent call last):
    ...
    NotFound: 404: Not Found
    '''
    path_parts = obj_path.split('/')
    if len(path_parts) <= 1:
        flask_abort(httplib.NOT_FOUND, 'Path {!r} is too short'.format(obj_path))
    return path_parts[:-1], path_parts[-1]

def get_object_by_path(obj_path):
    '''Finds an object by walking the object hierarchy. For example,
    a valid path can be:

        A/B/C/~/D/E/

    The search would go as follows:
    - Search for an object with `slug='A'` where `parent_id=None`.
    - Search for an object with `slug='B'` where `parent_id=A.id`.
    - Same for `C`.
    - Search for the type object of `C`.
    - Search for an object with `slug='D'` where `parent_id=C.type_id`.
    - Search for an object with `slug='E'` where `parent_id=D.id`.

    An object path must end with a '/' to point to the object itself.
    When an object's type supports actions, the action name should
    come after the last '/'. For example, the path for the 'labels'
    action of the X/Y/Z object is:

        X/Y/Z/labels

    '''
    logger.debug('Looking for object+action with path={!r}'.format(obj_path))
    cur_obj = None
    path_parts, action = _object_path_parts(obj_path)
    for part in path_parts:
        if part == '~':
            cur_obj = get_type_object(cur_obj)
        else:
            cur_obj = get_object_child(cur_obj, part)
        if cur_obj is None:
            flask_abort(httplib.NOT_FOUND, 'Could not find an object for {!r}'.format(part))
    # cur_obj must not be None. We must enter the for loop at least once
    # so we should call flask_abort if something is wrong.
    assert cur_obj is not None
    return cur_obj, action

def serialize_object(obj):
    return obj.as_dict()

#----------------------------------------------------------#
# Resources                                                #
#----------------------------------------------------------#

class ObjectTreeRoot(Resource):
    '''A resource for the root of the object hierarchy tree. Although
    all nodes in the object tree have the same behaviour, we give the
    root a special role as the place that stores all labs and where
    new labs can be created.
    '''
    def get(self):
        '''Returns all labs. In the object tree, the root is actually
        "null" so labs are the only objects with no parent.
        '''
        require_user()
        return dict(labs=list(serialize_object(lab) for lab in self._all_labs()))

    def _all_labs(self):
        return (obj for obj in get_object_children(None) if obj.has_type())

    create_lab_parser = RequestParser()
    create_lab_parser.add_argument('slug', required=True)
    create_lab_parser.add_argument('display_name', required=True)

    def _create_lab_type_object(self, slug):
        unique_lab_type_slug = str(uuid4())
        lab_type = Lab().create_type_object(parent=None, slug=unique_lab_type_slug)
        return lab_type

    def _create_lab(self, slug, display_name):
        if any(lab['slug'] == slug for lab in self._all_labs()):
            flask_abort(httplib.CONFLICT, "There's already a lab with this name")
        lab_type_obj = self._create_lab_type_object(slug)
        lab = create_object(
            parent       = None,
            type         = lab_type_obj,
            slug         = slug,
            display_name = display_name,
        )
        lab.save()
        return lab

    def post(self):
        require_admin()
        args = self.create_lab_parser.parse_args()
        lab = self._create_lab(args['slug'], args['display_name'])
        create_event(
            obj_id = lab.id,
            user_id = current_identity.id,
            interested_ids = [lab.id],
            title = 'Created the **{}** lab'.format(lab.display_name),
        )
        return serialize_object(lab), httplib.CREATED

class ObjectTreeNode(Resource):
    '''An object resource that allows for:
    - Accessing an object by its place in the object hierarchy
    - Invoking actions supported by the object type
    '''
    def _find_type_class(self, type_key):
        try:
            return all_type_classes[type_key]
        except LookupError as error:
            flask_abort(httplib.INTERNAL_SERVER_ERROR, error)

    def _find_and_invoke_from_type_class(self, type_class, get_action, obj, action_name):
        http_error = httplib.NOT_FOUND
        for attr in dir(type_class):
            action = getattr(type_class, attr)
            action_info = get_action(action)
            if action_info is None:
                continue
            if action_info['name'] == action_name:
                http_error = httplib.METHOD_NOT_ALLOWED
                if action_info['method'] == request.method:
                    return action(obj)
        flask_abort(http_error, 'Could not find handler for {!r}'.format(action_name))

    def invoke_type_action(self, type_obj, action_name):
        type_class = self._find_type_class(type_obj.type_key)
        return self._find_and_invoke_from_type_class(type_class, get_type_action, type_obj, action_name)

    def invoke_object_action(self, obj, action_name):
        type_obj = obj.get_type_object()
        type_class = self._find_type_class(type_obj.type_key)
        return self._find_and_invoke_from_type_class(type_class, get_object_action, obj, action_name)

    def invoke_action(self, path):
        obj, action_name = get_object_by_path(path)
        if obj.has_type():
            return self.invoke_object_action(obj, action_name)
        return self.invoke_type_action(obj, action_name)

    get    = invoke_action
    post   = invoke_action
    put    = invoke_action
    delete = invoke_action


def leer():
    global palabra
    palabra=(input('ingrese una frase:'))
def contar():
    vocales=['a','e','i','o','u']
    consosnantes=['b','c','d','f','g','h','j','k','l','m','n','ñ','p','r','s','t','v','w','x','y','z']
    contador=0
    contadorcons=0
    for o in palabra:
        for v in vocales:
            if(o==v):
                contador+=1
        for c in consosnantes:
            if(o==c):
                contadorcons +=1
    print('el numero devovales es:',contador)
    print('el numero de consonantes es:',contadorcons)

leer()
contar()


from __future__ import division, print_function
from sys import platform as _platform
import matplotlib
import matplotlib.cm as colormap

if _platform == "linux" or _platform == "linux2":
    matplotlib.use('Agg')
import matplotlib.pylab as plt
from graph_tool.all import *
import os
import matplotlib.cm as colormap
from matplotlib.colors import ColorConverter as color_converter
import pandas as pd
from . import printing
# import printing
import random
import datetime
import copy
import shutil
import numpy as np
import operator
import math
from scipy.sparse.linalg.eigen.arpack import eigsh as largest_eigsh
import scipy.stats as stats
from scipy.sparse import csr_matrix, issparse, lil_matrix, dok_matrix
import sys
from scipy.stats import powerlaw, poisson
from collections import defaultdict
import traceback
from .basics import create_folder_structure
import powerlaw as fit_powerlaw

def print_f(*args, **kwargs):
    if 'class_name' not in kwargs:
        kwargs.update({'class_name': 'gt_tools'})
    printing.print_f(*args, **kwargs)

def add_vertex_property(g, fn, p_name='weight', vertex_id_to_vertex=None, vertex_id_dtype='int', property_dtype='int',
                        sep=None, comment='#', col=1):
    pmap = g.new_vertex_property(property_dtype)
    if property_dtype is 'int':
        property_type_mapper = int
    elif property_dtype is 'float':
        property_type_mapper = float
    else:
        property_type_mapper = str

    if vertex_id_dtype is 'int' or vertex_id_to_vertex is None:
        vertex_id_mapper = int
    elif vertex_id_dtype is 'float':
        vertex_id_mapper = float
    else:
        vertex_id_mapper = str
    get_vertex = g.vertex if vertex_id_to_vertex is None else (lambda x: vertex_id_to_vertex[x])
    with open(fn, 'r') as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith(comment):
                line = line.split(sep)
                v = get_vertex(vertex_id_mapper(line[0]))
                p = property_type_mapper(line[col])
                pmap[v] = p
    g.vp[p_name] = pmap

def load_edge_list(filename, directed=False, vertex_weights=None, edge_weights=None, vertex_id_name='NodeId',
                   vertex_id_dtype='int', sep=None, comment='#', multiple_edges_per_line=False, store=True,
                   try_load_gt=True):
    """
    Loads an edge-list into a Graph (graph-tool) object and returns the Graph object.

    Attributes
    ----------
    filename : str
        the filename of the edge-list
    directed : bool
        treat the edge-list directed (True) or not (False)
    vertex_weights : dict
        dict where keys are the name of the resulting property-map and values are dicts containing properties of the
        weight like data-type etc. (cf. add_vertex_property)
        important: dict containing the properties must contain 'filename' : "filename_of_mapping"
        Example:
            vertex_weights = dict()
            one_v_weight = dict()
            one_v_weight['filename'] = 'foo.file'
            one_v_weight['property_dtype'] = 'int'  # you can use anything available in add_vertex_property()
            vertex_weights['MyVertexWeight'] = one_v_weight

    edge_weights : dict
        dict where keys are the name of the resulting property_map and values are tuples containing column id where
        the weight is in the edge-list and the data-type.
        Example:
            weight_name : (0, 'int')
            creates an integer property-map containing the values found in the first column AFTER src, tar.
    vertex_id_name : str
        Name of property map where the original vertex-name/-id... should be stored
    vertex_id_dtype : str
        Data-type of the original vertex-name/-id
    sep : str or None
        separator which should be used to split the lines of the edge-list file
    comment : str
        lines starting with this string are treated as comments (ignored)
    multiple_edges_per_line : bool
        Do the lines contain multiple-edges. (e.g., src tar1 tar2 tar3)
        If this is true, edge_weights are not supported.
    store : bool
        Should the graph be stored in .gt format?
    try_load_gt : bool
        Should the function try to load a previously stored .gt file? ("filename.gt")
        Note if there where modifications in the original edge-list, the function will reload the edge-list.

    Notes
    -----
    -

    Examples
    --------
    -
     """
    store_fname = filename + '.gt'
    if os.path.isfile(store_fname) and try_load_gt:
        try:
            g = load_graph(store_fname)
        except:
            print('failed loading. recreate graph')
            os.remove(store_fname)
            return load_edge_list(filename, directed=directed, vertex_id_dtype=vertex_id_dtype, sep=sep,
                                  comment=comment)
        if 'mtime' in g.gp.keys() and os.path.isfile(filename) and g.gp['mtime'] != os.path.getmtime(filename):
            print('modified edge-list. recreate graph')
            os.remove(filename + '.gt')
            return load_edge_list(filename, directed=directed, vertex_id_dtype=vertex_id_dtype, sep=sep,
                                  comment=comment)
        else:
            return g
    else:
        g = Graph(directed=directed)
        nodeid_to_v = defaultdict(g.add_vertex)
        edge_list = []
        v_type = int

        edge_weights_dtypes = dict()
        edge_weights_pmaps = dict()
        edge_weights_names = dict()
        if edge_weights is not None:
            for name, (col, dtype) in edge_weights.iteritems():
                edge_weights_names[col] = name
                edge_weights_pmaps[col] = g.new_edge_property(dtype)
                if dtype == 'int':
                    edge_weights_dtypes[col] = int
                elif dtype == 'float':
                    edge_weights_dtypes[col] = float
                else:
                    edge_weights_dtypes[col] = float
        src_target_extractor = (
            lambda x: x[:2]) if edge_weights is not None or not multiple_edges_per_line else (
            lambda x: x)
        weights_extractor = (lambda x: None) if edge_weights is None else (
            lambda x: tuple([edge_weights_dtypes[idx](i) for
                             idx, i in enumerate(x[2:])]))
        edge_weights_list = list()
        with open(filename, 'r') as f:
            for line in filter(lambda x: not x.startswith(comment), map(lambda x: x.strip(), f)):
                line = line.split(sep)
                nodes = src_target_extractor(line)
                try:
                    nodes = map(v_type, nodes)
                except ValueError:
                    v_type = float
                    try:
                        nodes = map(v_type, nodes)
                    except ValueError:
                        v_type = str
                        nodes = map(v_type, nodes)
                try:
                    src = int(nodeid_to_v[nodes[0]])
                except IndexError:
                    continue
                dest = map(lambda x: int(nodeid_to_v[x]), nodes[1:])
                edges = [(src, d) for d in dest]
                edge_weights_list.append(weights_extractor(line))
                edge_list.extend(edges)
        g.add_edge_list(edge_list)
        if edge_weights is not None:
            edges = list(map(lambda x: g.edge(*x), edge_list))
            for col_idx, col_weights in enumerate(zip(*edge_weights_list)):
                current_pmap = edge_weights_pmaps[col_idx]
                for e, w in zip(edges, col_weights):
                    current_pmap[e] = w
                g.ep[edge_weights_names[col_idx]] = current_pmap
        if vertex_id_dtype is not None:
            node_id_pmap = g.new_vertex_property(vertex_id_dtype)
            for v_id, v in nodeid_to_v.iteritems():
                node_id_pmap[v] = v_id
            g.vp[vertex_id_name] = node_id_pmap
        g.gp['filename'] = g.new_graph_property('string', filename)
        g.gp['mtime'] = g.new_graph_property('object', os.path.getmtime(filename))

        if vertex_weights is not None:
            for name, props in vertex_weights.iteritems():
                kwargs = dict()
                kwargs['vertex_id_to_vertex'] = nodeid_to_v
                kwargs['vertex_id_dtype'] = vertex_id_dtype
                kwargs['sep'] = sep
                kwargs['comment'] = comment
                fn = props['filename']
                props.pop('filename', None)
                kwargs.update(props)
                add_vertex_property(g, fn, p_name=name, **kwargs)
        if store:
            g.save(filename + '.gt', fmt='gt')
    return g

def net_from_adj(mat, directed=True, parallel_edges=True):
    g = Graph(directed=directed)
    assert mat.shape[0] == mat.shape[1]
    if not issparse(mat):
        mat = csr_matrix(mat)
    elif not isinstance(mat, csr_matrix):
        mat = mat.tocsr()
    g.add_vertex(mat.shape[0])
    w = None
    if not parallel_edges:
        if np.issubdtype(mat.dtype, int):
            w = g.new_edge_property('int')
        else:
            w = g.new_edge_property('float')
    row_idx, col_idx = mat.nonzero()
    data = np.array(mat.data)
    if not directed:
        # diag-upper part only (including diag)
        row_idx, col_idx, data = zip(*[(r, c, d) for r, c, d in zip(row_idx, col_idx, data) if c <= r])
    if parallel_edges:
        row_idx = np.array([r for r, d in zip(row_idx, data) for i in range(int(d))])
        col_idx = np.array([c for c, d in zip(col_idx, data) for i in range(int(d))])
    g.add_edge_list(zip(col_idx, row_idx))
    if w:
        w.a = data
        g.ep['weights'] = w
    return g

def load_property(network, filename, type='int', resolve='NodeId', sep=None, line_groups=False):
    assert isinstance(network, Graph)
    type = type.lower()
    if type == 'str' or type == 'string':
        pmap = network.new_vertex_property('string')
        mapper = lambda x: str(x)
    else:
        pmap = network.new_vertex_property(type)
        if type == 'int':
            mapper = lambda x: int(x)
        else:
            mapper = lambda x: float(x)
    if resolve is not None:
        res_map = network.vp[resolve]
        resolve = {str(res_map[v]): v for v in network.vertices()}
    mapped_vertices = set()
    com_id = 0
    with open(filename, 'r') as f:
        for line in filter(lambda l_line: not l_line.startswith('#'), f):
            line = line.strip().split(sep)
            if line_groups:
                line_vertices = []
                appender = line_vertices.append
                for i in line:
                    try:
                        if resolve is None:
                            i = network.vertex(int(i))
                        else:
                            i = resolve[i]
                        appender(i)
                    except KeyError:
                        pass
                for v in line_vertices:
                    pmap[v] = mapper(com_id)
                    mapped_vertices.add(int(v))
                com_id += 1
            else:
                try:
                    v = network.vertex(int(line[0])) if resolve is None else resolve[line[0]]
                    pmap[v] = mapper(line[1])
                    mapped_vertices.add(int(v))
                except KeyError:
                    pass
    unmapped_v = set(map(int,network.vertices())) - mapped_vertices
    if unmapped_v:
        print(filename, 'contained no mapping for', len(unmapped_v) / network.num_vertices() * 100, '% of all vertices')
        print('unmapped vertices:', list(unmapped_v)[:100])
    return pmap

def get_graph_com_connectivity(g, com_map='com'):
    com_map = g.vp[com_map]
    intern_edges = 0
    between_edges = 0
    for e in g.edges():
        if com_map[e.source()] == com_map[e.target()]:
            intern_edges += 1
        else:
            between_edges += 1
    num_edges = (intern_edges + between_edges)
    intern_edges /= num_edges
    between_edges /= num_edges
    return intern_edges, between_edges

def check_aperiodic(g):
    if isinstance(g, str):
        a = adjacency(load_graph(g))
        name = g.rsplit('/')[-1].replace('.gt', '')
        print('aperiodic:', name)
    else:
        a = adjacency(g)
    b = a * a
    diag_two_sum = b.diagonal().sum()
    print('\tA*A diag sum:', int(diag_two_sum))
    b *= a
    diag_three_sum = b.diagonal().sum()
    print('\tA*A*A diag sum:', int(diag_three_sum))
    aper = bool(diag_two_sum) and bool(diag_three_sum)
    print('\taperiodic:', aper)
    return aper

class SBMGenerator():
    @staticmethod
    def gen_stoch_blockmodel(num_nodes=1000, blocks=5, self_con=.97, other_con=0.03, directed=False,
                             degree_seq='powerlaw', powerlaw_exp=2.4, num_links=None, loops=False, min_degree=1,
                             con_prob_matrix=None, increase_lcc_prob=True, parallel_edges=False,
                             node_pick_strat=('dist', 'dist'), scale_con_prop_matrix=True):
        g = Graph(directed=directed)
        com_pmap = g.new_vertex_property('int')
        if isinstance(blocks, list):
            num_nodes = sum(blocks)
            num_blocks = len(blocks)
            block_sizes = blocks
        else:
            num_blocks = blocks
            block_sizes = [int(num_nodes / num_blocks) for i in range(num_blocks)]
            num_unmapped_nodes = num_nodes % num_blocks
            # print('unmapped nodes', num_unmapped_nodes)
            if num_unmapped_nodes > 0:
                for i in range(num_unmapped_nodes):
                    block_sizes[i] += 1
        g.add_vertex(num_nodes)

        if con_prob_matrix is not None:
            assert con_prob_matrix.shape[0] == con_prob_matrix.shape[1]
            con_prob_matrix /= con_prob_matrix.sum()
        com_pmap.a = np.hstack([np.array([idx] * i) for idx, i in enumerate(block_sizes)]).flatten()
        g.vp['com'] = com_pmap
        other_con /= ((num_blocks - 1) if num_blocks > 1 else 1)

        prob_pmap = g.new_vertex_property('float')
        block_to_vertices = dict()
        block_to_cumsum = dict()

        if isinstance(degree_seq, (np.ndarray, list, tuple)) and len(degree_seq) == num_nodes:
            degree_seq = degree_seq.astype('float')
            fixed_deg_seq = True
        else:
            fixed_deg_seq = False
            if degree_seq == 'powerlaw':
                degree_seq = stats.zipf.rvs(powerlaw_exp, loc=min_degree, size=num_nodes).astype('float')
            elif degree_seq == 'random':
                degree_seq = np.random.random(size=num_nodes)
            elif degree_seq == 'exp':
                degree_seq = np.random.exponential(size=num_nodes)
            else:
                degree_seq = np.array([1.0] * num_nodes)
            degree_seq.sort()
        degree_seq /= degree_seq.sum()
        multiplier = min_degree/degree_seq.min()
        degree_seq *= multiplier
        # print(degree_seq)
        if num_links is None:
            # print('min degree:', degree_seq.min())
            num_links = int(np.round((degree_seq.round().sum() / 2)))
            print('#links not set. using:', num_links)
        # print(degree_seq)
        block_deg_seq_sum = dict()
        vertices_array = np.array(list(map(int, g.vertices())))
        degree_indices = set(range(g.num_vertices()))
        for i in range(num_blocks):
            mask = com_pmap.a == i
            block_num_vertices = mask.sum()
            block_to_vertices[i] = vertices_array[mask]
            if fixed_deg_seq:
                block_deg_seq_idx = block_to_vertices[i]
            else:
                block_deg_seq_idx = random.sample(degree_indices, block_num_vertices)
                degree_indices -= set(block_deg_seq_idx)
            block_deg_seq = degree_seq[block_deg_seq_idx]
            deg_seq_sum = block_deg_seq.sum()
            block_deg_seq /= deg_seq_sum
            cum_sum = np.cumsum(block_deg_seq)
            assert np.allclose(cum_sum[-1], 1)
            block_to_cumsum[i] = cum_sum
            block_deg_seq_sum[i] = deg_seq_sum
            prob_pmap.a[mask] = block_deg_seq
        if scale_con_prop_matrix:
            blocks_prob = list()
            for i in range(num_blocks):
                row = list()
                for j in range(num_blocks):
                    if con_prob_matrix is None:
                        if i == j:
                            val = self_con
                        else:
                            val = other_con
                    else:
                        val = con_prob_matrix.item((i, j))
                    row.append(val * block_deg_seq_sum[i] * block_deg_seq_sum[j])
                blocks_prob.append(np.array(row))
            blocks_prob = np.array(blocks_prob)
            blocks_prob /= blocks_prob.sum()
        else:
            blocks_prob = con_prob_matrix
        # print(blocks_prob)
        cum_sum = np.cumsum(blocks_prob)
        assert np.isclose(cum_sum[-1], 1)
        # print(cum_sum)
        if parallel_edges:
            edges = list()
            edges_adder = edges.append
        else:
            edges = set()
            edges_adder = edges.add

        if isinstance(node_pick_strat, str):
            node_pick_strat = (node_pick_strat, node_pick_strat)
        pick_funcs = dict()
        get_rnd_node = SBMGenerator.get_random_node
        inv_prob = SBMGenerator.inverse_prob
        pick_funcs['rnd'] = lambda b, s=None: random.choice(block_to_vertices[b])
        pick_funcs['dist'] = lambda b, s=None: block_to_vertices[b][get_rnd_node(block_to_cumsum[b])]
        pick_funcs['invdist'] = lambda b, s=None: block_to_vertices[b][
            get_rnd_node(inv_prob(block_to_cumsum[b]))]
        pick_funcs['dist_other_com_inv_dist'] = lambda b, s=None: \
            block_to_vertices[b][get_rnd_node(block_to_cumsum[b])] \
            if (s is None or s == b) else \
                block_to_vertices[b][get_rnd_node(inv_prob(block_to_cumsum[b]))]

        # more efficient way to pick both inverse to the dist
        if node_pick_strat[0] == node_pick_strat[1] == 'invdist':
            node_pick_strat[0] = node_pick_strat[1] = 'dist'
            block_to_cumsum = {key: inv_prob(val) for key, val in block_to_cumsum.items()}

        src_pick_func, dest_pick_func = map(lambda x: pick_funcs[x], node_pick_strat)

        get_one_rnd_block = SBMGenerator.get_one_random_block
        if increase_lcc_prob:
            for v in g.vertices():
                if directed or v.out_degree() == 0:
                    src_block = com_pmap[v]
                    init_len = len(edges)
                    while init_len == len(edges):
                        dest_b = get_one_rnd_block(cum_sum, num_blocks, src_block) \
                            if 'dist_other_com_inv_dist' not in node_pick_strat else src_block
                        dest_v = dest_pick_func(dest_b, src_block)
                        link = (int(v), dest_v)
                        is_loop = v == dest_v
                        if not is_loop:
                            if not directed:
                                link = tuple(sorted(link))
                            edges_adder(link)
                        elif loops:
                            edges_adder(link)

        get_rnd_blocks = SBMGenerator.get_random_blocks
        for link_idx in range(num_links - len(edges)):
            while True:
                #maybe switch to: get random node. identify block. get random dest-block.
                src_b, dest_b = get_rnd_blocks(cum_sum, num_blocks)
                src_v = src_pick_func(src_b, dest_b)
                dest_v = dest_pick_func(dest_b, src_b)
                link = (src_v, dest_v)
                is_loop = src_v == dest_v
                if not is_loop:
                    if not directed:
                        link = tuple(sorted(link))
                    edges_adder(link)
                    break
                elif loops:
                    edges_adder(link)
                    break
        if not isinstance(edges, list):
            edges = list(edges)
        g.add_edge_list(edges)
        return g

    @staticmethod
    def inverse_prob(p_cum_sum):
        p = np.hstack([p_cum_sum[0], np.ediff1d(p_cum_sum)])
        p = 1. / p
        p /= p.sum()
        return p.cumsum()

    @staticmethod
    def get_random_node(cum_sum):
        rand_num = np.random.random()
        idx = np.searchsorted(cum_sum, rand_num)
        if idx == len(cum_sum):
            idx -= 1
        return idx

    @staticmethod
    def get_random_blocks(cum_sum, num_blocks):
        rand_num = np.random.random()
        idx = np.searchsorted(cum_sum, rand_num)
        row = int(idx / num_blocks)
        col = idx % num_blocks
        #print(rand_num)
        #print(row, col)
        #print(cum_sum)
        return row, col

    @staticmethod
    def get_one_random_block(cum_sum, num_blocks, row):
        src_b = None
        dest_b = None
        while src_b is None or row != src_b:
            src_b, dest_b = SBMGenerator.get_random_blocks(cum_sum, num_blocks)
        return dest_b

    @staticmethod
    def gen_bow_tie_model(scc_size, out_size, in_size, con_prob_matrix=None, **kwargs):
        blocks = [scc_size, out_size, in_size]
        if con_prob_matrix is None:
            con_prob_matrix = np.zeros((3, 3))
            # in-comp
            con_prob_matrix[0, 0] = 0.001  # self
            con_prob_matrix[0, 1] = 0.1  # scc
            con_prob_matrix[0, 2] = 0.05  # out

            # scc
            con_prob_matrix[1, 1] = 5  # self
            con_prob_matrix[1, 2] = 0.1  # out

            con_prob_matrix[2, 2] = 0.01  # self
        np.set_printoptions(formatter=dict(float=lambda x: '%.5f' % x))
        print(con_prob_matrix / con_prob_matrix.sum())
        return SBMGenerator.gen_stoch_blockmodel(blocks=blocks, con_prob_matrix=con_prob_matrix,
                                                 directed=True, **kwargs)

    @staticmethod
    def analyse_graph(g, filename='output/net', draw_net=False):
        print(str(g))
        deg_map = g.degree_property_map('total')
        plt.close('all')
        ser = pd.Series(deg_map.a)
        ser.plot(kind='hist', bins=int(deg_map.a.max()), lw=0)
        plt.xlabel('degree')
        plt.ylabel('num nodes')
        res = fit_powerlaw.Fit(deg_map.a, discrete=True)
        print('powerlaw alpha:', res.power_law.alpha)
        print('powerlaw xmin:', res.power_law.xmin)
        plt.title('powerlaw alpha:' + str(res.power_law.alpha) + ' || powerlaw xmin:' + str(res.power_law.xmin))
        plt.savefig(filename + '_degdist.png', bbox_tight=True)
        plt.close('all')
        if draw_net:
            graph_draw(g, vertex_fill_color=g.vp['com'], output_size=(200, 200),
                       vertex_size=prop_to_size(deg_map, mi=2, ma=15, power=1.), output=filename + '_network.png',
                       bg_color=[1, 1, 1, 1])

def bow_tie(graph):
    assert graph.is_directed()
    largest_component = label_largest_component(graph)
    weakly_components = label_components(graph, directed=False)[0]
    largest_component_corresponding_weakly = list(
        set(weakly_components.a[np.nonzero(largest_component.a)[0]]))
    assert len(largest_component_corresponding_weakly) == 1
    largest_component_corresponding_weakly = largest_component_corresponding_weakly[0]
    wcc = (weakly_components.a == largest_component_corresponding_weakly)

    # Core, In and Out
    all_nodes = set(range(graph.num_vertices()))
    scc = set(np.nonzero(largest_component.a)[0])
    scc_node = random.sample(scc, 1)[0]
    graph_reversed = GraphView(graph, reversed=True)

    outc = np.nonzero(label_out_component(graph, scc_node).a)[0]
    inc = np.nonzero(label_out_component(graph_reversed, scc_node).a)[0]
    outc = set(outc) - scc
    inc = set(inc) - scc

    # Tubes, Tendrils and Other
    wcc = set(np.nonzero(wcc)[0])
    tube = set()
    out_tendril = set()
    in_tendril = set()
    other = all_nodes - wcc
    remainder = wcc - inc - outc - scc

    for idx, r in enumerate(remainder):
        print(idx + 1, '/', len(remainder), end='\r')
        predecessors = set(np.nonzero(label_out_component(graph_reversed, r).a)[0])
        successors = set(np.nonzero(label_out_component(graph, r).a)[0])
        if any(p in inc for p in predecessors):
            if any(s in outc for s in successors):
                tube.add(r)
            else:
                in_tendril.add(r)
        elif any(s in outc for s in successors):
            out_tendril.add(r)
        else:
            other.add(r)

    vp_bowtie = graph.new_vertex_property('string')
    for component, label in [
        (inc, 'IN'),
        (scc, 'SCC'),
        (outc, 'OUT'),
        (in_tendril, 'TL_IN'),
        (out_tendril, 'TL_OUT'),
        (tube, 'TUBE'),
        (other, 'OTHER')
    ]:
        for node in component:
            vp_bowtie[graph.vertex(node)] = label
    graph.vp['bowtie'] = vp_bowtie

    bow_tie = map(len, [inc, scc, outc, in_tendril, out_tendril, tube, other])
    assert sum(bow_tie) == graph.num_vertices()
    bow_tie = [100 * x / graph.num_vertices() for x in bow_tie]
    bow_tie = dict(IN=bow_tie[0], SCC=bow_tie[1], OUT=bow_tie[2], TL_IN=bow_tie[3], TL_OUT=bow_tie[4], TUBE=bow_tie[5],
                   OTHER=bow_tie[6])
    return bow_tie

# Generator Class works with GraphTool generators, as they provide more functionality than NetworkX Generators
class GraphGenerator():
    # init generator
    def __init__(self, num_nodes=5000, directed=False):
        self.directed = directed
        self.num_nodes = num_nodes
        self.graph = None
        self.node_id_map = None
        self.return_and_reset()

    def return_and_reset(self):
        result = self.graph
        self.graph = Graph(directed=self.directed)
        self.node_id_map = defaultdict(lambda x: self.graph.add_vertex())
        return result

    # start creating blockmodel graph
    def create_blockmodel_graph(self, blocks=7, connectivity=10, model="blockmodel-traditional"):
        def corr(a, b):
            if a == b:
                return 0.999
            else:
                return 0.001

        self.print_f("Starting to create Blockmodel Graph with {} nodes and {} blocks".format(self.num_nodes, blocks))

        self.graph, vertex_colors = random_graph(self.num_nodes, lambda: poisson(connectivity), directed=False, model=model, block_membership=lambda: random.randint(1, blocks),
                                                 vertex_corr=corr)
        self.graph.vertex_properties["colorsComm"] = vertex_colors
        return self.return_and_reset()

    def create_fully_connected_graph(self, size=1000, directed=False, self_edges=False):
        return self.create_stochastic_blockmodel_graph(blocks=1, size=size, directed=directed, self_edges=self_edges, self_block_connectivity=1.0, other_block_connectivity=1.0)

    @staticmethod
    def create_sbm_lined_up_matrix(blocks=10, self_block_connectivity=None, other_block_connectivity=None):
        if self_block_connectivity is None:
            self_block_connectivity = [0.9]
        elif isinstance(self_block_connectivity, (int, float)):
            self_block_connectivity = [self_block_connectivity]
        if other_block_connectivity is None:
            other_block_connectivity = [0.1]
        elif isinstance(other_block_connectivity, (int, float)):
            other_block_connectivity = [other_block_connectivity]
        connectivity_matrix = []
        blocks_range = range(blocks)
        for idx in blocks_range:
            row = []
            outer_prob = other_block_connectivity[idx % len(other_block_connectivity)]
            inner_prob = self_block_connectivity[idx % len(self_block_connectivity)]
            for jdx in blocks_range:
                if idx != jdx:
                    row.append(outer_prob / pow(abs(idx - jdx), 2))
                else:
                    row.append(inner_prob)
            connectivity_matrix.append(row)
        return connectivity_matrix

    # scale = None
    # scale = relative
    # scale = absolute
    def create_stochastic_blockmodel_graph(self, blocks=10, size=100, self_block_connectivity=0.9, other_block_connectivity=0.1, connectivity_matrix=None, directed=False,
                                           self_edges=False, power_exp=None, scale=None, plot_stat=False):
        size = size if isinstance(size, list) else [size]
        self_block_connectivity = self_block_connectivity if isinstance(self_block_connectivity, list) else [self_block_connectivity]
        other_block_connectivity = other_block_connectivity if isinstance(other_block_connectivity, list) else [other_block_connectivity]

        num_nodes = sum([size[i % len(size)] for i in range(blocks)])
        if power_exp is None:
            self.print_f("Starting to create Stochastic Blockmodel Graph with {} nodes and {} blocks".format(num_nodes, blocks))
        else:
            self.print_f("Starting to create degree-corrected (alpha=" + str(power_exp) + ") Stochastic Blockmodel Graph with {} nodes and {} blocks".format(num_nodes, blocks))
        self.print_f('convert/transform probabilities')
        blocks_range = range(blocks)
        block_sizes = np.array([size[i % len(size)] for i in blocks_range])

        # create connectivity matrix of self- and other-block-connectivity
        if connectivity_matrix is None:
            connectivity_matrix = []
            self.print_f('inner conn: ' + str(self_block_connectivity) + '\tother conn: ' + str(other_block_connectivity))
            for idx in blocks_range:
                row = []
                for jdx in blocks_range:
                    if idx == jdx:
                        row.append(self_block_connectivity[idx % len(self_block_connectivity)])
                    else:
                        if scale is not None:
                            prob = other_block_connectivity[idx % len(other_block_connectivity)] / (num_nodes - block_sizes[idx]) * block_sizes[jdx]
                            if directed:
                                row.append(prob)
                            else:
                                row.append(prob / 2)
                        else:
                            row.append(other_block_connectivity[idx % len(other_block_connectivity)])
                connectivity_matrix.append(row)

        # convert con-matrix to np.array
        if connectivity_matrix is not None and isinstance(connectivity_matrix, np.matrix):
            connectivity_matrix = np.asarray(connectivity_matrix)

        # convert con-matrix to np.array
        if connectivity_matrix is not None and not isinstance(connectivity_matrix, np.ndarray):
            connectivity_matrix = np.array(connectivity_matrix)

        self.print_f('conn mat')
        printing.print_matrix(connectivity_matrix)

        if scale == 'relative' or scale == 'absolute':
            new_connectivity_matrix = []
            for i in blocks_range:
                connectivity_row = connectivity_matrix[i, :] if connectivity_matrix is not None else None
                nodes_in_src_block = block_sizes[i]
                multp = 1 if scale == 'absolute' else (nodes_in_src_block * (nodes_in_src_block - 1))
                row_prob = [(connectivity_row[idx] * multp) / (nodes_in_src_block * (nodes_in_block - 1)) for idx, nodes_in_block in enumerate(block_sizes)]
                new_connectivity_matrix.append(np.array(row_prob))
            connectivity_matrix = np.array(new_connectivity_matrix)
            self.print_f(scale + ' scaled conn mat:')
            printing.print_matrix(connectivity_matrix)

        # create nodes and store corresponding block-id
        self.print_f('insert nodes')
        vertex_to_block = []
        appender = vertex_to_block.append
        colors = self.graph.new_vertex_property("float")
        for i in range(blocks):
            block_size = size[i % len(size)]
            for j in range(block_size):
                appender((self.graph.add_vertex(), i))
                node = vertex_to_block[-1][0]
                colors[node] = i

        # create edges
        get_rand = np.random.random
        add_edge = self.graph.add_edge

        self.print_f('create edge probs')
        degree_probs = defaultdict(lambda: dict())
        for vertex, block_id in vertex_to_block:
            if power_exp is None:
                degree_probs[block_id][vertex] = 1
            else:
                degree_probs[block_id][vertex] = math.exp(power_exp * np.random.random())

        tmp = dict()
        self.print_f('normalize edge probs')
        all_prop = []
        for block_id, node_to_prop in degree_probs.iteritems():
            sum_of_block_norm = 1 / sum(node_to_prop.values())
            tmp[block_id] = {key: val * sum_of_block_norm for key, val in node_to_prop.iteritems()}
            all_prop.append(tmp[block_id].values())
        degree_probs = tmp
        if plot_stat:
            plt.clf()
            plt.hist(all_prop, bins=15)
            plt.savefig("prop_dist.png", bbox_tight=True)
            plt.close('all')

        self.print_f('count edges between blocks')
        edges_between_blocks = defaultdict(lambda: defaultdict(int))
        for idx, (src_node, src_block) in enumerate(vertex_to_block):
            conn_mat_row = connectivity_matrix[src_block, :]
            for dest_node, dest_block in vertex_to_block:
                if get_rand() < conn_mat_row[dest_block]:
                    edges_between_blocks[src_block][dest_block] += 1

        self.print_f('create edges')
        for src_block, dest_dict in edges_between_blocks.iteritems():
            self.print_f(' -- Processing Block {}. Creating links to: {}'.format(src_block, dest_dict))
            for dest_block, num_edges in dest_dict.iteritems():
                self.print_f('   ++ adding {} edges to {}'.format(num_edges, dest_block))
                for i in range(num_edges):
                    # find src node
                    prob = np.random.random()
                    prob_sum = 0
                    src_node = None
                    for vertex, v_prob in degree_probs[src_block].iteritems():
                        prob_sum += v_prob
                        if prob_sum >= prob:
                            src_node = vertex
                            break
                    # find dest node
                    prob = np.random.random()
                    prob_sum = 0
                    dest_node = None
                    for vertex, v_prob in degree_probs[dest_block].iteritems():
                        prob_sum += v_prob
                        if prob_sum >= prob:
                            dest_node = vertex
                            break
                    if src_node is None or dest_node is None:
                        print('Error selecting node:', src_node, dest_node)
                    if self.graph.edge(src_node, dest_node) is None:
                        if self_edges or not src_node == dest_node:
                            add_edge(src_node, dest_node)
        self.graph.vertex_properties["colorsComm"] = colors
        return self.return_and_reset()

    def create_preferential_attachment(self, communities=10):
        self.graph = price_network(self.num_nodes, directed=False, c=0, gamma=1, m=1)
        self.graph.vertex_properties['colorsComm'] = community_structure(self.graph, 1000, communities)
        return self.return_and_reset()

    # add node to graph and check if node is in node_dict
    def add_node(self, node_id, further_mappings=None):
        v = self.node_id_map[node_id]
        self.graph.vp['NodeId'][v] = node_id
        if further_mappings is not None:
            assert isinstance(further_mappings, dict)
            for key, val in further_mappings.iteritems():
                self.graph.vp[key][v] = val
        return v

    def load_smw_collab_network(self, filename, communities=10):
        self.print_f("Creating Graph")
        id_prop = self.graph.new_vertex_property("int")
        self.graph.vp["label"] = self.graph.new_vertex_property("string")

        f = open(filename, "rb")
        for idx, line in enumerate(f):
            if idx % 1000 == 0:
                self.print_f("--> parsing line %d" % idx)
            split_line = line.strip("\n").split("\t")
            source_v = self.add_node(split_line[0], id_prop)
            if split_line[1] != "":
                target_v = self.add_node(split_line[1], id_prop)
                self.graph.add_edge(source_v, target_v)

        self.print_f("Detecting Communities")
        self.graph.vp['colorsComm'] = community_structure(self.graph, 1000, communities)
        remove_self_loops(self.graph)
        remove_parallel_edges(self.graph)
        return self.return_and_reset()

    @staticmethod
    def increment_neighbours(vertices, b):
        for n in vertices:
            b[int(n)] += 1

    # start creating random graph
    # NOTE:
    # If min_degree is too small, graph will be disconnected and consist of many smaller graphs!
    # This could make diffusion problematic!
    def create_random_graph(self, min_degree=2, max_degree=40, model="probabilistic", communities=10):
        # Function to sample edges between nodes!
        self.print_f('create random graph')

        def sample_k(min_val, max_val, k=None):
            accept = False
            while not accept:
                k = random.randint(min_val, max_val + 1)
                accept = random.random() < 1.0 / k
            return k

        self.graph = random_graph(self.num_nodes, lambda: sample_k(min_degree, max_degree), model=model, vertex_corr=lambda i, k: 1.0 / (1 + abs(i - k)), directed=self.directed,
                                  n_iter=100)
        self.graph.vp['colorsComm'] = community_structure(self.graph, 10000, max_degree / communities)
        return self.return_and_reset()

    # start loading  graph
    def create_karate_graph(self):
        self.graph = collection.data["karate"]
        # Removing descriptions and readme, as they screw with the GML parser of networkx!
        self.graph.gp['description'] = ''
        self.graph.gp['readme'] = ''
        # Calculating Colors and updating members
        self.graph.vp['colorsComm'] = community_structure(self.graph, 10000, 2)
        self.directed = self.graph.is_directed()
        self.num_nodes = self.graph.num_vertices()
        return self.return_and_reset()

    def loaded_post_action(self):
        self.directed = self.graph.is_directed()
        self.num_nodes = self.graph.num_vertices()
        self.print_f("Graph loaded with {} nodes and {} edges".format(self.graph.num_vertices(), self.graph.num_edges()))

    # load graph from gml
    def load_gml(self, fn):
        self.print_f("Loading GML")
        self.graph = load_graph(fn)
        self.loaded_post_action()
        return self.return_and_reset()

    # load graph from file
    def load_gt(self, fn):
        self.print_f("Loading GT")
        self.graph = load_graph(fn)
        self.loaded_post_action()
        return self.return_and_reset()

    @staticmethod
    def print_f(*args, **kwargs):
        kwargs.update({'class_name': 'GraphGenerator'})
        print_f(*args, **kwargs)

def calc_eigenvalues(graph, num_ev=100):
    num_ev = min(100, num_ev)
    print_f("Extracting adjacency matrix!")
    adj_mat = adjacency(graph, weight=None)
    print_f("Starting calculation of {} Eigenvalues".format(num_ev))
    evals_large_sparse, evecs_large_sparse = largest_eigsh(adj_mat, num_ev * 2, which='LM')
    print_f("Finished calculating Eigenvalues")
    weights = sorted([float(x) for x in evals_large_sparse], reverse=True)[:num_ev]
    graph.gp["top_eigenvalues"] = graph.new_graph_property("vector<float>", weights)
    return graph

def cleanup_graph(graph, largest_comp=True, parallel_edges=False, self_loops=False):
    if largest_comp:
        reduce_to_largest_component(graph)
    if not parallel_edges:
        remove_parallel_edges(graph)
    if not self_loops:
        remove_self_loops(graph)
    return graph

def reduce_to_largest_component(graph):
    print_f("Reducing graph to largest connected component!")
    l = label_largest_component(graph)
    graph = GraphView(graph, vfilt=l)
    graph.purge_vertices(in_place=True)
    return graph

def calc_vertex_properties(graph, max_iter_ev=1000, max_iter_hits=1000):
    print_f("Calculating PageRank")
    graph.vp["pagerank"] = pagerank(graph)

    print_f("Calculating Clustering Coefficient")
    graph.vp["clustercoeff"] = local_clustering(graph)

    print_f("Calculating Eigenvector Centrality")
    ev, ev_centrality = eigenvector(graph, weight=None, max_iter=max_iter_ev)
    graph.vp["evcentrality"] = ev_centrality

    print_f("Calculating HITS")
    eig, authorities, hubs = hits(graph, weight=None, max_iter=max_iter_hits)
    graph.vp["authorities"] = authorities
    graph.vp["hubs"] = hubs

    print_f("Calculating Degree Property Map")
    graph.vertex_properties["degree"] = graph.degree_property_map("total")
    return graph

'''
def fast_sd(g, src_ids=None, dest_ids=None, pairs=None, max_dist=10, loops=False):
    # print(g)
    all_vertices_ids = list(map(int, g.vertices()))
    if src_ids is None:
        src_ids = all_vertices_ids
    elif not hasattr(src_ids, '__iter__'):
        src_ids = [int(src_ids)]
    if dest_ids is None:
        dest_ids = all_vertices_ids
    elif not hasattr(dest_ids, '__iter__'):
        dest_ids = [int(dest_ids)]
    if pairs is not None:
        if isinstance(pairs, list):
            pairs = set(pairs)
        elif isinstance(pairs, tuple):
            pairs = {pairs}
    elif g.is_directed():
        pairs = {(src, dest) for src in src_ids for dest in dest_ids if src != dest}
    else:
        pairs = {(src, dest) for src in src_ids for dest in dest_ids if src < dest}

    assert isinstance(pairs, set)

    A = adjacency(g).astype(bool)
    M = A.copy()
    M_old = None
    distances = defaultdict(lambda: defaultdict(int))
    m_max = g.num_vertices() * (g.num_vertices() - 1)
    reachable = 0
    for current_distance in range(1, max_dist + 1):
        m_non_zero = M.nnz

        if m_non_zero == 0:
            break
        reachable += m_non_zero
        print(current_distance, reachable/m_max)
        #pairs_with_current_distance = pairs & set(zip(*m_non_zero))
        #for src, dest in pairs_with_current_distance:
        #    distances[dest][src] = current_distance
        #pairs -= pairs_with_current_distance
        if not pairs:
            # print('break at:', current_distance)
            break
        M_old = M.copy()
        M *= A
        M -= M_old
        M = M.astype(bool)
    if len(pairs):
        # print('#pairs without distance:', len(pairs))
        pass
    return distances
'''


'''
使用递归+回溯，需要注意的是合法 ip 的判断，即：

每一节的数组均在 0 到 255 之间，包含 0 和 255
001，01 均是不合法的，但 0 ，10，100合法
共有4节，不多不少
'''

class Solution:
    """
    @param s: the IP string
    @return: All possible valid IP addresses
    """

    def restoreIpAddresses(self, s):
        ips = []
        self.dfs(s, 0, ips, '')
        return ips

    def dfs(self, s, sub, ips, ip):
        if sub == 4:
            if s == '':
                ips.append(ip[1:])
                return
        for i in xrange(1, 4):
            if i <= len(s):
                if int(s[:i]) <= 255:
                    self.dfs(s[i:], sub + 1, ips, ip + '.' + s[:i])
                if s[0] == '0':
                    break


import pandas as pd

import inspect
import pytest

from functools import wraps

from xbbg.core import utils, assist
from xbbg.io import files, logs, storage, cached

try:
    import pdblp
except ImportError as imp_err:
    print(imp_err)
    pdblp = utils.load_module(f'{files.abspath(__file__)}/pdblp.py')

_CON_SYM_ = '_xcon_'
_PORT_, _TIMEOUT_ = 8194, 30000

if hasattr(pytest, 'config'):
    if 'with_bbg' not in pytest.config.option:
        pytest.skip('no Bloomberg')
    if not pytest.config.option.with_bbg:
        pytest.skip('no Bloomberg')

def with_bloomberg(func):
    """
    Wrapper function for Bloomberg connection

    Args:
        func: function to wrap
    """
    @wraps(func)
    def wrapper(*args, **kwargs):

        scope = utils.func_scope(func=func)
        param = inspect.signature(func).parameters
        port = kwargs.pop('port', _PORT_)
        timeout = kwargs.pop('timeout', _TIMEOUT_)
        restart = kwargs.pop('restart', False)
        all_kw = {
            k: args[n] if n < len(args) else v.default
            for n, (k, v) in enumerate(param.items()) if k != 'kwargs'
        }
        all_kw.update(kwargs)
        log_level = kwargs.get('log', logs.LOG_LEVEL)

        for to_list in ['tickers', 'flds']:
            conv = all_kw.get(to_list, None)
            if hasattr(conv, 'tolist'):
                all_kw[to_list] = getattr(conv, 'tolist')()
            if isinstance(conv, str):
                all_kw[to_list] = [conv]

        cached_data = []
        if scope in ['xbbg.blp.bdp', 'xbbg.blp.bds']:
            to_qry = cached.bdp_bds_cache(func=func.__name__, **all_kw)
            cached_data += to_qry.cached_data

            if not (to_qry.tickers and to_qry.flds):
                if not cached_data: return pd.DataFrame()
                res = pd.concat(cached_data, sort=False).reset_index(drop=True)
                if not all_kw.get('raw', False):
                    res = assist.format_output(
                        data=res, source=func.__name__,
                        col_maps=all_kw.get('col_maps', dict())
                    )
                return res

            all_kw['tickers'] = to_qry.tickers
            all_kw['flds'] = to_qry.flds

        if scope in ['xbbg.blp.bdib']:
            data_file = storage.hist_file(
                ticker=all_kw['ticker'], dt=all_kw['dt'], typ=all_kw['typ'],
            )
            if files.exists(data_file):
                logger = logs.get_logger(func, level=log_level)
                if all_kw.get('batch', False): return
                logger.debug(f'reading from {data_file} ...')
                return assist.format_intraday(data=pd.read_parquet(data_file), **all_kw)

        _, new = create_connection(port=port, timeout=timeout, restart=restart)
        res = func(**{
            k: v for k, v in all_kw.items() if k not in ['raw', 'col_maps']
        })
        if new: delete_connection()

        if scope.startswith('xbbg.blp.') and isinstance(res, list):
            final = cached_data + res
            if not final: return pd.DataFrame()
            res = pd.DataFrame(pd.concat(final, sort=False))

        if (scope in ['xbbg.blp.bdp', 'xbbg.blp.bds']) \
                and (not all_kw.get('raw', False)):
            res = assist.format_output(
                data=res.reset_index(drop=True), source=func.__name__,
                col_maps=all_kw.get('col_maps', dict()),
            )

        return res
    return wrapper

def create_connection(port=_PORT_, timeout=_TIMEOUT_, restart=False):
    """
    Create Bloomberg connection

    Returns:
        (Bloomberg connection, if connection is new)
    """
    if _CON_SYM_ in globals():
        if not isinstance(globals()[_CON_SYM_], pdblp.BCon):
            del globals()[_CON_SYM_]

    if (_CON_SYM_ in globals()) and (not restart):
        con = globals()[_CON_SYM_]
        if getattr(con, '_session').start(): con.start()
        return con, False

    else:
        con = pdblp.BCon(port=port, timeout=timeout)
        globals()[_CON_SYM_] = con
        con.start()
        return con, True

def delete_connection():
    """
    Stop and destroy Bloomberg connection
    """
    if _CON_SYM_ in globals():
        con = globals().pop(_CON_SYM_)
        if not getattr(con, '_session').start(): con.stop()


from django.http import HttpResponseRedirect
from requests import post
from .forms import Mailing_listForm, ClientForm, MessageForm, DirectoryClient, AddMailing_listForm
from .models import Mailing_list, Client, Message
from django.shortcuts import render, redirect, get_object_or_404
from django.views.generic import ListView, DetailView
from twilio.rest import Client
import pickle

class ClientDetailView(DetailView):
    model = Client

    def createClient(request):
        if request.method == "POST":
            client = ClientForm(request.POST)
            if client.is_valid():
                client.save()
                return HttpResponseRedirect('#')
            else:
                error = 'Форма была не верной'

        form = ClientForm
        context = {
            'form': form,
            'error': error,
        }
        return render(request, '#', context)

    def editClient(request, pk):
        contact = get_object_or_404(Client, pk=pk)
        client = ClientForm(request.POST or None, instance=post)
        if client.is_valid():
            client.save()
            return redirect('#')

        context = {
            'client': client,
            'contact': contact,
        }
        return render(request, context)

    def deleteClient(request, pk):
        contact = get_object_or_404(Client, pk=pk)
        if request.method == 'POST':
            contact.delete()
            return redirect('#')

        context = {
            'object': contact,
        }
        return render(request, context)

class Mailing_listView(ListView):

    def get_detail(request):
        import pickle
        qs = Mailing_list.objects.values_list('id_res', 'title', 'date_start', 'date_end')
        return render(request, qs)

    def object_detail(request, status):
        content = MessageForm.objects.filter(category__name__contains=status)

        context = {
            'content': content,
            'status': status,
        }
        return render(request, context)

class CreateDeleteMailing_listView(DetailView):
    model = Mailing_list

    def createMailing_list(request):
        if request.method == "POST":
            content = Mailing_listForm(request.POST)
            if content.is_valid():
                content.save()
                return HttpResponseRedirect('#')
            else:
                error = 'Форма была не верной'

        form = Mailing_listForm
        context = {
            'form': form,
            'error': error,
        }
        return render(request, '#', context)

    def deleteMailing_list(request, pk):
        Ml = get_object_or_404(Client, pk=pk)
        if request.method == 'POST':
            Ml.delete()
            return redirect('#')

        context = {
            'object': Ml,
        }
        return render(request, context)

class SendingMeiling_list(DetailView):
    model = Mailing_listForm, Client

    def send_sms(request):
        if request.method == 'POST':
            text = Mailing_listForm.objects.values_list('title')
            client = Client.objects.values_list('code' + 'tel_number')
            teg = Client.objects.values_list('teg')
            start = Mailing_listForm.objects.values_list('date_start')
            end = Mailing_listForm.objects.values_list('date_end')
            try:
                if start is not end:
                    auth_token = '...'

                    sender = Client(auth_token)

                    message = sender.messages.create(
                        body=text,
                        from_=sender,
                        to=client or teg,
                    )
                    return 'Successfully sent!'
            except Exception as ex:
                return 'Error sending', ex


import tweepy
from decouple import config
import urllib
import re
import arxiv
import pandas as pd
import notion_df

notion_df.pandas()
notion_df.config(api_key=config('NOTION_TOKEN'))
page_url = config('reading_list_url')

def run():
    # get notion table to update to
    df_download = pd.read_notion(page_url, resolve_relation_values=True)

    # get twitter data
    client = tweepy.Client(
        bearer_token=config('bearer_token'),
        consumer_key=config('client_id'),
        consumer_secret=config('client_secret'),
        access_token=config('api_key'),
        access_token_secret=config('api_key_secret'),
        wait_on_rate_limit=True
    )

    response = client.get_user(username=config('twitter_username'))
    user_id = response.data.data['id']
    response = client.get_liked_tweets(user_id,
                                       max_results=100,
                                       tweet_fields=["attachments", "context_annotations"])

    get_tweets = 1
    while get_tweets != 0:

        for tweet in response.data:
            # TO DO - doesn't search sub tweets
            tweet_urls = re.findall("http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", tweet.text)
            if not tweet_urls:
                continue
            for url in tweet_urls:
                try:
                    opener = urllib.request.build_opener()
                    request = urllib.request.Request(url)
                    url_response = opener.open(request)
                    actual_url = url_response.geturl()
                    extracted_url = str(actual_url)
                except:
                    extracted_url = str(url)

                # only finds arxiv results
                if 'arxiv' not in extracted_url:
                    continue
                else:
                    print(f'link_to_paper: {extracted_url}')
                    search_id = extracted_url.split("https://arxiv.org/abs/")[1]
                    paper = next(arxiv.Search(id_list=[search_id]).results())
                    title = paper.title

                if set([title]).issubset(df_download['title']):
                    get_tweets = 0
                    break
                else:
                    print(f'Uploading to Notion: {title}')
                    new_df = pd.DataFrame({
                        'title': title,
                        'source': extracted_url,
                        'type': ['research paper']})

                    new_df.to_notion(page_url, resolve_relation_values=True)
                    #filename = f'{title}.pdf'
                    #paper.download_pdf(dirpath="./papers", filename=filename)

            if get_tweets == 0:
                break
            else:
                if response.meta['result_count'] != 0:
                    response = client.get_liked_tweets(user_id,
                                                       max_results=100,
                                                       pagination_token=response.meta['next_token'],
                                                       tweet_fields=["attachments", "context_annotations"])

    return

if __name__ == '__main__':
    run()