#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This module initializes the VPCs necessary for the rest of cloud formation

from itertools import chain

from troposphere import Parameter, Ref, Tags, Join, Output, Select, FindInMap
import troposphere.ec2 as ec2

import config as cfn
from config import CLOUDNAME

def emit_configuration():
    # Build the VPC here
    template = cfn.template

    # Parameters here
    nat_instance_class = template.add_parameter(
        Parameter(
            'NatInstanceType', Type='String', Default='t2.micro',
            Description='NAT instance type',
            AllowedValues=cfn.usable_instances(),
            ConstraintDescription='Instance size must be a valid instance type'
        )
    )

    keyname_param = template.add_parameter(
        Parameter(
            'KeyName', Type='AWS::EC2::KeyPair::KeyName',
            Description='Name of an existing EC2 KeyPair to enable SSH access into machines'
        )
    )

    cfn.keyname = keyname_param

    vpc = template.add_resource(
        ec2.VPC(
            'VPC',
            CidrBlock='{0}.0.0/16'.format(cfn.CIDR_PREFIX),
            EnableDnsSupport=True,
            EnableDnsHostnames=True,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, Ref('AWS::Region')]))
        )
    )

    # Add the VPC to cloudformation
    cfn.vpcs.append(vpc)

    gateway = template.add_resource(
        ec2.InternetGateway(
            'InternetGateway',
            Tags=Tags(Name='InternetGateway-{0}'.format(CLOUDNAME)),
            DependsOn=vpc.title
        )
    )

    gateway_attachment = template.add_resource(
        ec2.VPCGatewayAttachment(
            'GatewayAttachment',
            VpcId=Ref(vpc),
            InternetGatewayId=Ref(gateway),
            DependsOn=gateway.title
        )
    )

    public_routing_table = template.add_resource(
        ec2.RouteTable(
            '{0}PublicRouteTable'.format(cfn.VPC_NAME),
            VpcId=Ref(vpc),
            DependsOn=vpc.title
        )
    )

    # Add in the public route through the gateway
    template.add_resource(
        ec2.Route(
            'PublicRoute',
            RouteTableId=Ref(public_routing_table),
            DestinationCidrBlock=cfn.DEFAULT_ROUTE,
            GatewayId=Ref(gateway),
            DependsOn=gateway_attachment.title
        )
    )

    # Define the Security Group for the NATs
    nat_ingress_rules = [
        ec2.SecurityGroupRule(
            IpProtocol='tcp', CidrIp=cfn.DEFAULT_ROUTE, FromPort=p, ToPort=p
        ) for p in [22, 80, 443, 11371]
    ]

    nat_egress_rules = [
        ec2.SecurityGroupRule(
            IpProtocol='-1', CidrIp=cfn.DEFAULT_ROUTE, FromPort=0, ToPort=65535,
        )
    ]

    nat_security_group = template.add_resource(
        ec2.SecurityGroup(
            'NATSecurityGroup',
            GroupDescription='Security Group for NAT instances',
            VpcId=Ref(vpc),
            SecurityGroupIngress=nat_ingress_rules,
            SecurityGroupEgress=nat_egress_rules,
            DependsOn=vpc.title
        )
    )

    platform_subnets = list()
    master_subnets = list()
    public_subnets = list()
    vpn_subnets = list()
    worker_subnets = list()
    database_subnets = list()
    subnet_identifier = 'private'

    for idx, zone in enumerate(cfn.get_availability_zones()):
        region = Ref('AWS::Region')
        full_region_descriptor = Join('', [region, zone])

        # create a public subnet in each availability zone
        public_subnet = template.add_resource(
            ec2.Subnet(
                '{0}PublicSubnet{1}'.format(cfn.VPC_NAME, zone),
                VpcId=Ref(vpc),
                CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 80 + idx),
                AvailabilityZone=full_region_descriptor,
                DependsOn=vpc.title,
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'public-subnet', full_region_descriptor]))
            )
        )
        public_subnets.append(public_subnet)

        # Associate the public routing table with the subnet
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}PublicRouteTableAssociation'.format(public_subnet.title),
                SubnetId=Ref(public_subnet),
                RouteTableId=Ref(public_routing_table),
                DependsOn=public_subnet.title
            )
        )

        # Create the NAT instance in the public subnet
        nat_name = '{0}Nat{1}'.format(cfn.VPC_NAME, zone)
        nat_instance = template.add_resource(
            ec2.Instance(
                nat_name,
                DependsOn=vpc.title,
                InstanceType=Ref(nat_instance_class),
                KeyName=Ref(keyname_param),
                SourceDestCheck=False,
                ImageId=FindInMap('RegionMap', region, int(cfn.Amis.NAT)),
                NetworkInterfaces=[
                    ec2.NetworkInterfaceProperty(
                        Description='Network interface for {0}'.format(nat_name),
                        GroupSet=[Ref(nat_security_group)],
                        SubnetId=Ref(public_subnet),
                        AssociatePublicIpAddress=True,
                        DeviceIndex=0,
                        DeleteOnTermination=True
                    )
                ],
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'nat', full_region_descriptor]))
            )
        )

        # Associate the private routing table with the NAT
        private_routing_table = template.add_resource(
            ec2.RouteTable(
                '{0}PrivateRouteTable{1}'.format(cfn.VPC_NAME, zone),
                VpcId=Ref(vpc),
                DependsOn=nat_instance.title,
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'private-route-table', full_region_descriptor]))
            )
        )

        template.add_resource(
            ec2.Route(
                'PrivateRoute{0}'.format(zone.upper()),
                RouteTableId=Ref(private_routing_table),
                DestinationCidrBlock=cfn.DEFAULT_ROUTE,
                InstanceId=Ref(nat_instance),
                DependsOn=private_routing_table.title
            )
        )

        worker_subnet = ec2.Subnet(
            '{0}{1}WorkerSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/22'.format(cfn.CIDR_PREFIX, 100 + (idx * 4)),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-worker-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        worker_subnets.append(worker_subnet)

        platform_subnet = ec2.Subnet(
            '{0}{1}PlatformSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 10 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-platform-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        platform_subnets.append(platform_subnet)

        master_subnet = ec2.Subnet('{0}{1}MasterSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 90 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-master-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        master_subnets.append(master_subnet)

        vpn_subnet = ec2.Subnet(
            '{0}VpnSubnet{1}'.format(cfn.VPC_NAME, zone),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 60 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('.', [full_region_descriptor, cfn.CLOUDNAME, cfn.CLOUDENV, "vpn-client"]))
        )
        vpn_subnets.append(vpn_subnet)

        database_subnet = ec2.Subnet(
            '{0}DatabaseSubnet{1}'.format(cfn.VPC_NAME, zone),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 20 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, subnet_identifier, 'database-subnet', full_region_descriptor]))
        )
        database_subnets.append(database_subnet)

    # Associate a routing table with each of the master/platform/worker subnets
    if cfn.USE_PRIVATE_SUBNETS:
        routing_table = private_routing_table
    else:
        routing_table = public_routing_table

    for sn in chain(worker_subnets, master_subnets, platform_subnets, database_subnets):
        template.add_resource(sn)
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}{1}RouteTableAssociation'.format(sn.title, subnet_identifier),
                SubnetId=Ref(sn),
                RouteTableId=Ref(routing_table),
                DependsOn=sn.title
            )
        )

    # associate the public routing table with the VPN subnets
    for sn in vpn_subnets:
        template.add_resource(sn),
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}RouteTableAssociation'.format(sn.title),
                SubnetId=Ref(sn),
                RouteTableId=Ref(public_routing_table),
                DependsOn=sn.title
            )
        )

    # affect the global state
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.PLATFORM, platform_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.PUBLIC, public_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.MASTER, master_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.WORKER, worker_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.VPN, vpn_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.DATABASE, database_subnets)


#-*- coding: UTF-8 -*-
#查询器
import time
from selenium.webdriver.support.select import Select
from WebServiceLayer.ticketelement.ticketWait import now_show,not_show

class querierlist():
    def add_querier(self,driver,customname,queriername,querierdescription):
        driver.find_element_by_css_selector()
        driver.find_element_by_xpath("//a[contains(text(),'工单查询器')]").click()
        #进入iframe id="rightContentIfr"
        driver.switch_to.frame(driver.find_element_by_css_selector("iframe#rightContentIfr"))
        time.sleep(2)
        not_show(driver,"[class='glyphicon glyphicon-plus']")
        driver.find_element_by_css_selector("[class='glyphicon glyphicon-plus']").click()
        text5=driver.find_element_by_css_selector("[class='glyphicon glyphicon-plus']").text
        print(text5)
        now_show(driver,"[name='ticketName']")
        now_show(driver,"[name='ticketDescription']")
        driver.find_element_by_css_selector("[name='ticketName']").click()
        driver.find_element_by_css_selector("[name='ticketName']").send_keys(queriername)
        driver.find_element_by_css_selector("[name='ticketDescription']").click()
        driver.find_element_by_css_selector("[name='ticketDescription']").send_keys(querierdescription)
        now_show(driver,"[id='RequiredPOVItem']")
        #判断满足任意以下条件中有添加的文本字段
        driver.find_element_by_css_selector("[name='RequiredPSelect[0]']").click()
        s=driver.find_element_by_css_selector("[name='RequiredPSelect[0]']")
        Select(s).select_by_visible_text(customname)#通过文本值定位
        #[name="OptionalPSelect[0]"]
        #判断满足任意以下条件中有添加的文本字段
        s = driver.find_element_by_css_selector("[name='OptionalPSelect[0]']")
        Select(s).select_by_visible_text(customname)  # 通过文本值定位
        #等待元素出现#form-group_container1 > div:nth-last-child(1)
        target_elem1 = driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)")
        driver.execute_script("return arguments[0].scrollIntoView();", target_elem1)
        #获取最后文本的值是什么
        wenbenshuju=driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>span:nth-child(2)").text
        try:
            assert wenbenshuju==customname
            print("选择正确")
        except Exception as e:
            print(e)
        #获取这个选择字段的类型
        webenleixing=driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>span:nth-child(3)")
        try:
            assert wenbenshuju == "文本"
            print("类型正确")
        except Exception as e:
                print(e)
        #选取这个文本字段在查询器中展示
        driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>input[type='checkbox']").click()
        driver.find_element_by_css_selector("[id='selectColumnSubmint']").click()
        #获取视图选择后的判断
        wenbenshuju1=driver.find_element_by_css_selector("tbody[class='info']>tr:last-child>td:first-child").text
        try:
            assert wenbenshuju1 == customname
            print("类型正确")
        except Exception as e:
                print(e)
        webenleixing1=driver.find_element_by_css_selector("tbody[class='info'']>tr:last-child>td:nth-child(2)").text
        try:
            assert webenleixing1 == "文本"
            print("类型正确")
        except Exception as e:
                print(e)
        driver.find_element_by_css_selector("[name='btnSubmit']").click()
        time.sleep(5)
        chaxunqiname=driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child(1)>td:nth-child(2)").text
        querierdescriptionname=driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child(1)>td:nth-child(3)").text
        try:
            assert chaxunqiname == queriername
            print("查询器标题保存没问题")
        except Exception as e:
                print(e)
        time.sleep(5)
        try:
            assert querierdescriptionname == querierdescription
            print("查询器描述保存没问题")
        except Exception as e:
                print(e)
        time.sleep(5)
        #获取获取查询器的个数  现在只获取数字  但是现在获取的是全部的信息
        querier = driver.find_element_by_css_selector("#page-wrapper>div>div:nth-child(2)>table>tbody>tr:nth-child(1)> td").text
        queriercount=querier[7:8]#这里得再调整一下
        print (queriercount)
        #获取活动的查询器的名称 进行遍历对比
        for index in range(int(queriercount.strip())):
            # print (index + 1)
            #这里得做一次处理取出错不包括空格的数据 但是现在获取的是有空格的信息
            querier_value = driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child('+ str(index+1)+')>td:nth-child(2)").text
            service_winid_value = querier_value.split('(')[1].split(')')[0]
            try:
                assert service_winid_value ==chaxunqiname
            except Exception as e:
                print(e)
                print("新建的查询器未出现在活动的列表中")

        # 获取失效的查询器的个数  只获取数字  但是现在获取的是全部的信息
        querier1 = driver.find_element_by_css_selector("#page-wrapper>div>div:nth-child(2)>table>tbody>tr:nth-child(1)> td").text
        queriercount1 = querier1[7:8]  # 这里得再调整一下
        # 获取非活动的查询器的名称 进行遍历对比
        for index in range(int(queriercount1.strip())):
                # print (index + 1)
                # 这里得做一次处理取出错不包括空格的数据 但是现在获取的是有空格的信息
            querier_value = driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child('+ str(index+1)+')>td:nth-child(2)").text
            service_winid_value = querier_value.split('(')[1].split(')')[0]
            try:
                assert service_winid_value != chaxunqiname
            except Exception as e:
                print(e)
                print("新建的查询器出现在非活动的列表中")

    def shixiao_querier(self, driver, customname, queriername, querierdescription):
        driver.find_element_by_css_selector


# -*- coding: utf-8 -*-
import socket

print('Creando socket - Cliente')

# armamos el socket los parámetros que recibe el socket indican el tipo de conexión
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# lo conectamos al puerto acordado
client_socket.connect(('localhost', 8888))

# mandamos un mensajito
print("... Mandamos cositas")

# socket debe recibir bytes, por lo que encodeamos el mensaje
send_message = 'Holi c:'.encode()

# enviamos el mensaje a través del socket
client_socket.send(send_message)
print("... Mensaje enviado")

# y esperamos una respuesta
message = client_socket.recv(1024)
print(' -> Respuesta del servidor: <<' + message.decode() + '>>')

# cerramos la conexión
client_socket.close()


import matplotlib.pyplot as plt
import imp
from sklearn.linear_model import LinearRegression
import numpy as np
x=[[11,23],[22,33],[33,55]]
y=[[4],[6],[7]]
x1=np.array(x)
print(x1)
x2=np.array(y)
model=LinearRegression()
model.fit(x,y)
print("现在是展示Xi为(5,5)的时候，5预测出的y的值的大小：",model.predict([[5,10]]))#这里预测的格式十分重要，前面怎么洗的中括号，后面也怎么写
print(x+y)
#下面我们对我们的模型进行评估，这也是机器学习当中十分重要的一步了。
x_test=[[11,23],[24,35],[34,60]]
y_test=[[4],[5],[8]]
print("现在看看我们模型的准确度了：",model.score(x_test,y_test))

sklearn/多元线性回归（超级简单）.py

多元线性回归（超级简单）.py


# PROTOSPACER PRECISION LASSO
# Tristan Henser-Brownhill, September, 2018
# For Chakrabarti et al.

# IMPORTS
import os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.linear_model import LassoCV, Lasso
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error, mean_absolute_error, median_absolute_error
from sklearn.externals import joblib
from scipy.stats.stats import pearsonr, spearmanr, linregress, kendalltau # scipy v. 1.0.0
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from time import time
from scipy.stats import randint as sp_randint

#DATA PREPROCESSING

# import
data = pd.read_csv(os.path.join('data','random_train_set80pc.csv'))
data_test = pd.read_csv(os.path.join('data','random_test_set20pc.csv'))
#data_ob = pd.read_csv(os.path.join('data','ob_all48h.csv'))
data_ob = pd.read_csv(os.path.join('data','ob_HCT116_48h.csv'))
#data_ob = pd.read_csv(os.path.join('data','ob_HEK293_48h.csv'))
#data_ob = pd.read_csv(os.path.join('data','ob_K562_48h.csv'))

# prepare the data
df_train_x = data.iloc[:,2:] #extract pixels; all rows, all except sgRna name (0) and label (1)
y_train = data.iloc[:,1].values #extract labels
df_test_x = data_test.iloc[:,2:]
y_test = data_test.iloc[:,1].values
df_ob_x = data_ob.iloc[:,2:]
y_ob = data_ob.iloc[:,1].values

# one hot encoding

# Note label order encoded ['A','C','G','T']
df_train_x = df_train_x.apply(LabelEncoder().fit_transform)
df_test_x = df_test_x.apply(LabelEncoder().fit_transform)
df_ob_x = df_ob_x.apply(LabelEncoder().fit_transform)
enc = OneHotEncoder(sparse = False)
enc.fit(df_train_x)
x_train = enc.transform(df_train_x)
x_test = enc.transform(df_test_x)
x_ob = enc.transform(df_ob_x)
'''
#MAIN MODEL SETUP

# parameter setup and model fitting
las = LassoCV(alphas = np.logspace(-5, 2, 30), cv = 10, verbose = 1, n_jobs = 4).fit(x_train, y_train) #fit the pixel values (x) to the labels (y)

print("Chosen alpha:", las.alpha_)

# save model
joblib.dump(las, os.path.join('saved_models','lasso_model.pkl'))
'''

las = joblib.load(os.path.join('saved_models','lasso_model.pkl'))
# MAKE PREDICTIONS AND CHECK ACCURACY

# get predictions
train_labels = y_train
train_pred = las.predict(x_train)
test_labels = y_test
test_pred = las.predict(x_test)
ob_labels = y_ob
ob_pred = las.predict(x_ob)

train_slope, train_intercept, train_r_value, train_p_value, train_std = linregress(train_labels, train_pred)
test_slope, test_intercept, test_r_value, test_p_value, test_std = linregress(test_labels, test_pred)
ob_slope, ob_intercept, ob_r_value, ob_p_value, ob_std = linregress(ob_labels, ob_pred)

# print metrics

with open('lasso_stats.csv', "w") as csv_file:

    csv_file.write('{0}{1}{2}{3}{4}{5}{6}{7}{8}'.format("SET,","MSE,","MAE,","MAD,","LR_slope,","LR_intercept,","LR_r_val,","LR_wald_p_val,","LR_stdev\n"))
    csv_file.write('{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}{10}{11}{12}{13}{14}{15}{16}'.format("TRAIN,",mean_squared_error(train_labels, train_pred),",",mean_absolute_error(train_labels, train_pred),",",median_absolute_error(train_labels, train_pred),",",train_slope,",",train_intercept,",",train_r_value,",",train_p_value,",",train_std,"\n"))
    csv_file.write('{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}{10}{11}{12}{13}{14}{15}{16}'.format("TEST,",mean_squared_error(test_labels, test_pred),",",mean_absolute_error(test_labels, test_pred),",",median_absolute_error(test_labels, test_pred),",",test_slope,",",test_intercept,",",test_r_value,",",test_p_value,",",test_std,"\n"))
    csv_file.write('{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}{10}{11}{12}{13}{14}{15}'.format("OVERBEEK,",mean_squared_error(ob_labels, ob_pred),",",mean_absolute_error(ob_labels, ob_pred),",",median_absolute_error(ob_labels, ob_pred),",",ob_slope,",",ob_intercept,",",ob_r_value,",",ob_p_value,",",ob_std))

print('TRAINING SET RESULTS:')
print('MSE: ', mean_squared_error(train_labels, train_pred))
print('MAE: ', mean_absolute_error(train_labels, train_pred))
print('Median AE: ', median_absolute_error(train_labels, train_pred))
print('LR:', linregress(train_labels, train_pred))
print('Pearson:', pearsonr(train_labels, train_pred))
print('Spearman:', spearmanr(train_labels, train_pred))
print('Kendall:', kendalltau(train_labels, train_pred))

print('\nTEST SET RESULTS:')
print('MSE: ', mean_squared_error(test_labels, test_pred))
print('MAE: ', mean_absolute_error(test_labels, test_pred))
print('Median AE: ', median_absolute_error(test_labels, test_pred))
print('LR:', linregress(test_labels, test_pred))
print('Pearson:', pearsonr(test_labels, test_pred))
print('Spearman:', spearmanr(test_labels, test_pred))
print('Kendall:', kendalltau(test_labels, test_pred))

print('\nOVERBEEK RESULTS:')
print('MSE: ', mean_squared_error(ob_labels, ob_pred))
print('MAE: ', mean_absolute_error(ob_labels, ob_pred))
print('Median AE: ', median_absolute_error(ob_labels, ob_pred))
print('LR:', linregress(ob_labels, ob_pred))
print('Pearson:', pearsonr(ob_labels, ob_pred))
print('Spearman:', spearmanr(ob_labels, ob_pred))
print('Kendall:', kendalltau(ob_labels, ob_pred))

# FEATURE IMPORTANCES

importances = las.coef_
nt_importance = []
allAs = []
allCs = []
allGs = []
allTs = []
As = []
Cs = []
Gs = []
Ts = []
negAs = []
negCs = []
negGs = []
negTs = []
# output |summed| coeffs
for i in range(80):
    if ((i+1)%4 == 0):
        # sum of the abs of coeffs from the individual one-hot encoded features
        nt_importance.append(abs(importances[i-3])+abs(importances[i-2])+abs(importances[i-1])+abs(importances[i]))

        allAs.append(importances[i-3])
        if importances[i-3] < 0:
            As.append(0)
            negAs.append(importances[i-3])
        else:
            As.append(importances[i-3])
            negAs.append(0)

        allCs.append(importances[i-2])
        if importances[i-2] < 0:
            Cs.append(0)
            negCs.append(importances[i-2])
        else:
            Cs.append(importances[i-2])
            negCs.append(0)

        allGs.append(importances[i-1])
        if importances[i-1] < 0:
            Gs.append(0)
            negGs.append(importances[i-1])
        else:
            Gs.append(importances[i-1])
            negGs.append(0)

        allTs.append(importances[i])
        if importances[i] < 0:
            Ts.append(0)
            negTs.append(importances[i])
        else:
            Ts.append(importances[i])
            negTs.append(0)

        # output table of individual nt letter at nt pos contributions
nt_importance.append(abs(importances[80])+abs(importances[81])+abs(importances[82])+abs(importances[83]))

allAs.append(importances[80])
if importances[80] < 0:
    As.append(0)
    negAs.append(importances[80])
else:
    As.append(importances[80])
    negAs.append(0)

allCs.append(importances[81])
if importances[81] < 0:
    Cs.append(0)
    negCs.append(importances[81])
else:
    Cs.append(importances[81])
    negCs.append(0)

allGs.append(importances[82])
if importances[82] < 0:
    Gs.append(0)
    negGs.append(importances[82])
else:
    Gs.append(importances[82])
    negGs.append(0)

allTs.append(importances[83])
if importances[83] < 0:
    Ts.append(0)
    negTs.append(importances[83])
else:
    Ts.append(importances[83])
    negTs.append(0)

nt_importance.append(abs(importances[84]))
allAs.append(0), allCs.append(0), allGs.append(0), allTs.append(0)
nt_importance.append(abs(importances[85]))
allAs.append(0), allCs.append(0), allGs.append(0), allTs.append(0)

# DRAW PLOTS

#feature coefficients vs alphas
alphas = las.alphas_
coefs = []
for a in alphas:
    check_las = Lasso(alpha=a)
    check_las.fit(x_train, y_train)
    coefs.append(check_las.coef_)

ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])  # reverse axis
plt.xlabel('alpha', fontsize = 18)
plt.ylabel('coefficients',fontsize = 18)
plt.yticks(fontsize = 12)
plt.xticks(fontsize = 12)
plt.title('LASSO coefficients as a function alpha', fontsize = 18)
plt.axis('tight')
plt.tight_layout()
plt.savefig('lasso_coeff_lambda.pdf', dpi = 300)

# Lasso model selection
m_log_alphas = -np.log10(las.alphas_)
plt.figure()
plt.plot(m_log_alphas, las.mse_path_, ':')
plt.plot(m_log_alphas, las.mse_path_.mean(axis=-1), 'k',
         label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(las.alpha_), linestyle='--', color='k',
            label='alpha: CV estimate')
plt.legend()
plt.xlabel('-log(alpha)', fontsize = 18)
plt.ylabel('MSE', fontsize = 18)
plt.yticks(fontsize = 12)
plt.xticks(fontsize = 12)
plt.title('LASSO model selection', fontsize = 18)
plt.axis('tight')
plt.tight_layout()
plt.savefig('lasso_model_selection_cv.pdf', dpi = 300)

# plot the feature importance of LASSO (all)

plt.figure()
ind = np.arange(21)    # the x locations for the groups
width = 0.8       # the width of the bars: can also be len(x) sequence
p1 = plt.bar(ind, As, width, color = 'green')
p2 = plt.bar(ind, Ts, width, bottom = As, color = 'red')
p3 = plt.bar(ind, Cs, width, bottom = np.add(As, Ts), color = 'blue')
p4 = plt.bar(ind, Gs, width, bottom = np.add(Cs, np.add(As, Ts)), color = 'orange')
p5 = plt.bar(ind, negAs, width, color = 'green')
p6 = plt.bar(ind, negTs, width, bottom = negAs, color = 'red')
p7 = plt.bar(ind, negCs, width, bottom = np.add(negAs, negTs), color = 'blue')
p8 = plt.bar(ind, negGs, width, bottom = np.add(negCs, np.add(negAs, negTs)), color = 'orange')
plt.legend((p1[0], p2[0], p3[0], p4[0]), ('A', 'T', 'C', 'G'), fontsize  = 14, loc = 2)
plt.title('LASSO nucleotide importances', fontsize = 18)
plt.ylabel('coefficients', fontsize = 18)
plt.ylim([-0.1,0.2])
plt.yticks(fontsize = 14)
plt.xticks(range(23), ['-20', '-19', '-18', '-17', '-16', '-15', '-14', '-13', '-12', '-11', '-10', '-9', '-8', '-7', '-6', '-5', '-4', '-3', '-2', '-1', 'N', 'G', 'G'], rotation = 60, fontsize = 14)
plt.tight_layout()
plt.savefig('lasso_coeff.pdf', dpi = 300)

# plot the feature importances of LASSO (|summed| coeff)
plt.figure()
plt.title('LASSO nucleotide importances', fontsize = 18)
plt.bar(range(23), nt_importance, width = 0.8, color='gray')
plt.ylabel('absolute coefficients', fontsize = 18)
plt.yticks(fontsize = 12)
plt.xticks(range(23), ['-20', '-19', '-18', '-17', '-16', '-15', '-14', '-13', '-12', '-11', '-10', '-9', '-8', '-7', '-6', '-5', '-4', '-3', '-2', '-1', 'N', 'G', 'G'], rotation = 60, fontsize = 14)
plt.tight_layout()
plt.savefig('summed_lasso_coeff.pdf', dpi = 300)

##################

# Setting the positions and width for the bars
pos = list(range(len(allAs)))
width = 0.25
x_labels = ['-20', '-19', '-18', '-17', '-16', '-15', '-14', '-13', '-12', '-11', '-10', '-9', '-8', '-7', '-6', '-5', '-4', '-3', '-2', '-1', 'N', 'G', 'G']

# Plotting the bars
fig, ax = plt.subplots(figsize=(10,5))

# Create a bar with pre_score data,
# in position pos,
plt.bar(pos,
        #using df['pre_score'] data,
        allAs,
        # of width
        width,
        # with color
        color='green')

# Create a bar with mid_score data,
# in position pos + some width buffer,
plt.bar([p + width for p in pos],
        #using df['mid_score'] data,
        allTs,
        # of width
        width,
        # with color
        color='red')

# Create a bar with post_score data,
# in position pos + some width buffer,
plt.bar([p + width*2 for p in pos],
        #using df['post_score'] data,
        allCs,
        # of width
        width,
        # with color
        color='blue')

plt.bar([p + width*3 for p in pos],
        #using df['post_score'] data,
        allGs,
        # of width
        width,
        # with color
        color='orange')

# Set the y axis label
ax.set_ylabel('coefficient', fontsize = 20)

# Set the chart's title
ax.set_title('Nucleotide importance (LASSO)', fontsize = 20)

# Set the labels for the x ticks
plt.xticks(range(23), x_labels, rotation = 60, fontsize = 14)

# Set the position of the x ticks
ax.set_xticks([p + 2 * width for p in pos])

# Setting the x-axis and y-axis limits
plt.xlim(min(pos)-width, max(pos)+width*4)

# Adding the legend and showing the plot
plt.legend(['A', 'T', 'C', 'G'], loc='upper right', framealpha = 0, fontsize = 14)

for p in range(22):
    plt.axvline(p + width*3.5, color = 'black', linewidth = 0.3)

plt.savefig('trail_lasso_coeff.pdf', dpi = 600)

##################

# Plot pred vs observed (test)

plot_text = '{0}{1:.2f}{2}{3}{4:.2E}'.format('R-value: ', test_r_value,'\n','P-value: ', test_p_value)

fig2, ax2 = plt.subplots()
ax2.plot(test_labels, test_pred,'o')
ax2.plot(test_labels, test_intercept + test_slope*test_labels, 'r')
ax2.text(0.3,0.85,plot_text,fontsize=18)
ax2.set_title('Unseen test_data\n', fontsize = 18)
ax2.set_ylim([0.0,1.0])
ax2.set_xlim([0.0,1.0])
ax2.tick_params('both', labelsize = 18)
ax2.set_ylabel('Predicted precision', fontsize = 18)
ax2.set_xlabel('Observed precision', fontsize = 18)
ax2.set_aspect('equal', 'box')
fig2.tight_layout()
plt.savefig('lasso_predictions_vs_labels_test.pdf', dpi = 300)

# Plot pred vs observed (Overbeek)

plot_text = '{0}{1:.2f}{2}{3}{4:.2E}'.format('R-value: ', ob_r_value,'\n','P-value: ', ob_p_value)

fig2, ax2 = plt.subplots()
ax2.plot(ob_labels, ob_pred,'o', color = 'y')
ax2.plot(ob_labels, ob_intercept + ob_slope*ob_labels, 'r')
ax2.text(0.3,0.85,plot_text,fontsize=18)
ax2.set_title('HCT116 (van Overbeek et al)\n', fontsize = 18)
ax2.set_ylim([0.0,1.0])
ax2.set_xlim([0.0,1.0])
ax2.tick_params('both', labelsize = 18)
ax2.set_ylabel('Predicted precision', fontsize = 18)
ax2.set_xlabel('Observed precision', fontsize = 18)
ax2.set_aspect('equal', 'box')
fig2.tight_layout()
plt.savefig('lasso_predictions_vs_labels_overbeek.pdf', dpi = 300)


from bs4 import BeautifulSoup
import requests
import pandas as pd

result = {'date' : [],
          'title' : [],
          'content' : []}

print('수집 시작')

for j in range(2):
    p = requests.get('https://www.inven.co.kr/board/lostark/5352?p=' + str(j))
    soup = BeautifulSoup(p.text, 'html.parser')

    selection = soup.select('a.subject-link')

    for i in range(len(selection)):
        title = selection[i].text.replace("\xa0[밸런스]\xa0\xa0","")
        url = selection[i].get('href')

        q = requests.get(url)
        soup2 = BeautifulSoup(q.text, 'html.parser')

        content = soup2.select('div#powerbbsContent')[0].text
        date = soup2.select('div.articleDate')[0].text

        result['date'].append(date)
        result['title'].append(title)
        result['content'].append(content)

print('수집 완료')

final = pd.DataFrame(result)
final.to_excel('balance_text.xlsx', index = False)

print('파일 저장 완료')


# Make sure you are using the following environment to run this shell
# Python 3.5.2
# Requests 2.12.4

import requests
import re

url = 'http://www.pythonchallenge.com/pc/def/linkedlist.php'
parameters = {"nothing": 12345}
pattern = re.compile(r'[0-9]+$')
new_number = 0

# while parameters['nothing'] != new_number:
while True:
    ret = requests.get(url, params = parameters)
    res = pattern.search(ret.text)
    if not res:
        if ret.text == "Yes. Divide by two and keep going.":
            parameters['nothing'] = int(parameters['nothing']) / 2
            continue
        else:
            print(ret.text) # well, the result here is "peak.html"
            break
    new_number = res.group(0)
    parameters['nothing'] = new_number


from collections import deque

for _ in range(10):
    t = int(input())

    arr = deque(map(int, input().split()))

    while 1:
        for i in range(1,6):
            a = arr.popleft()
            if (a-i) <= 0:
                arr.append(0)
                print('#%d' %(t), end=' ')
                for j in arr:
                    print(j, end= ' ')
                print()
                break
            else:
                arr.append(a-i)
        if arr[-1] == 0:
            break

python daily coding/2020.6.25 (SW Expert Academy)/1225번 (암호생성기).py


import psycopg2

from EstoqueGauss.RequestDb import RequestDb

class ReadCsv:

    def __init__(self):
        self.connectDb = RequestDb()

    def openCsv(self):
        dic = {}

        usar = self.connectDb.getSapObjectFromDB()
        with open('C:/Users/matheus.bertho/Downloads/data.csv', 'r', encoding='utf-8') as r:
            arq = r.read().split('\n')
            for linhaFull in arq:
                print(linhaFull)
                itemLinha = linhaFull.split(',')
                if itemLinha == ['']:
                    continue
                try:
                    cod_fornecedor = itemLinha[2]
                    stock = int(itemLinha[3])
                    # print(stock)
                    dic[cod_fornecedor] = stock
                except:
                    continue
        for bat in usar:
            # print('\n')
            # print(bat.sap + '|' + bat.cod + '|' + str(bat.stock) + '|' + str(bat.newStock))
            try:
                bat.newStock = self._regra(dic.get(bat.cod))
            except:
                stockSAP = 4
                bat.newStock = 0 + stockSAP
            # print(self.regra(bat.newStock))
            # print(bat.sap + '|' + bat.cod + '|' + str(bat.stock) + '|' + str(bat.newStock))
            #
            # print('\n')

        return usar

    def _regra(self, stock):
        if stock > 500:
            return 500
        elif stock >= 5:
            return 500
        elif 5 > stock > 0:
            return stock
            # add regra buscar no SAP e somar
        elif stock == 0:
            return 0
            # add regra buscar no SAP e somar
        else:
            raise ValueError(stock)

# r = ReadCsv()
# r.openCsv()
# # r.test()


__author__ = 'jim'

from scrapy.http import Request

from alascrapy.items import ProductItem, CategoryItem, ReviewItem
from alascrapy.spiders.base_spiders.ala_spider import AlaSpider
from alascrapy.lib.generic import get_full_url, date_format

class DartyComSpider(AlaSpider):
    name = 'darty_com'
    allowed_domains = ['darty.com']
    start_urls = ['http://darty.com/']

    def parse(self, response):
        sub_category_urls = self.extract_list(response.xpath(
                "//ul[@class='level-2']/li/a[contains(@href,'nav')]/@href"))

        for sub_category_url in sub_category_urls:
            yield Request(url=get_full_url(response, sub_category_url), callback=self.parse_sub_category)

    def parse_sub_category(self, response):
        category_urls = self.extract_list(response.xpath(
                '//li[@class="visible" or @class="supplementaire"]/a[contains(@href,"achat") or contains(@href,"cat=")]/@href'))

        for category_url in category_urls:
            yield Request(url=get_full_url(response, category_url), callback=self.parse_category)

    def parse_category(self, response):
        container_style = response.xpath("//div[contains(@class, 'product_detail')]")

        if container_style:
            category = CategoryItem()
            category['category_path'] = self.extract_all(
                    response.xpath('//div[@id="header-breadcrumb-zone"]//a/text()'), " > ")
            category['category_leaf'] = self.extract(response.xpath(
                    '//div[@id="header-breadcrumb-zone"]//li[last()]/a/text()'))
            category['category_url'] = response.url
            yield category

            if not self.should_skip_category(category):
                filtered_category_url = response.url+'?p=200&avis=100'
                request = Request(url=filtered_category_url, callback=self.parse_filtered_category)
                request.meta['ocn'] = category['category_path']
                yield request

    def parse_filtered_category(self, response):
        next_page_link = self.extract(response.xpath("//div[@id='main_pagination_bottom']//a[contains(text(), 'Page suivante')]/@href"))
        if next_page_link:
            next_page_link = get_full_url(response, next_page_link)
            request = Request(url=next_page_link, callback=self.parse_filtered_category)
            request.meta['ocn'] = response.meta['ocn']
            yield request

        product_urls = self.extract_list(response.xpath('//h2/a/@href'))
        for product_url in product_urls:
            product_url = get_full_url(response, product_url)
            request = Request(url=product_url, callback=self.parse_product)
            request.meta['ocn'] = response.meta['ocn']
            yield request

    def parse_product(self, response):
        product = ProductItem()

        product['TestUrl'] = response.url
        product['OriginalCategoryName'] = response.meta['ocn']
        product['ProductName'] = self.extract_all(response.xpath('//div[@itemprop="name" or contains(@class,"product_name")]/span//text()'))
        if not product['ProductName']:
            product['ProductName'] = self.extract_all(response.xpath('//div[@itemprop="name" or contains(@class,"product_name")]//text()'))
        product['PicURL'] = self.extract(response.xpath('//img[@itemprop="image"]/@src'))
        product['ProductManufacturer'] = self.extract(response.xpath('//a[@id="darty_product_brand"]/text()'))
        product['source_internal_id'] = self.extract(response.xpath('//@data-comparator-codic'))
        yield product

        reviews = response.xpath('//div[contains(@class,"bloc_reviews_item_full")]')

        for review in reviews:
            user_review = ReviewItem()
            user_review['DBaseCategoryName'] = "USER"
            user_review['ProductName'] = product['ProductName']
            user_review['TestUrl'] = product['TestUrl']
            user_review['source_internal_id'] = product['source_internal_id']
            date = self.extract(review.xpath('.//span/text()'))
            user_review['TestDateText'] = date_format(date.strip('Avis soumis le '), '%d/%m/%Y')
            user_review['SourceTestRating'] = self.extract(review.xpath('.//div[@class="bloc_reviews_note"]/text()[1]'))
            user_review['Author'] = self.extract(review.xpath('.//b/text()'))
            user_review['TestTitle'] = self.extract(review.xpath('.//h3/text()'))
            user_review['TestSummary'] = self.extract_all(review.xpath('.//p[contains(@class,"text")]//text()'))
            yield user_review


#BBN_LICENSE_START -- DO NOT MODIFY BETWEEN LICENSE_{START,END} Lines
# Copyright (c) <2017,2018,2019,2020,2021>, <Raytheon BBN Technologies>
# To be applied to the DCOMP/MAP Public Source Code Release dated 2018-04-19, with
# the exception of the dcop implementation identified below (see notes).
#
# Dispersed Computing (DCOMP)
# Mission-oriented Adaptive Placement of Task and Data (MAP)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#BBN_LICENSE_END
#!/usr/bin/env python3

import warnings
with warnings.catch_warnings():
    import re
    import sys
    import argparse
    import os
    import os.path
    import logging
    import logging.config
    import json
    from pathlib import Path
    import pandas as pd
    import csv
    import matplotlib.pyplot as plt
    import map_utils
    import multiprocessing

script_dir=os.path.abspath(os.path.dirname(__file__))

"""
Statistics for the whole system have a pid of -1
"""
system_stats_pid = -1

def get_logger():
    return logging.getLogger(__name__)

def bytes_to_gigabytes(b):
    return b / 1024 / 1024 / 1024

def pid_label(pid, pid_names):
    return f"{pid_names[pid]}_{pid}"

def make_pid_key(pid_names):
    """
    Create a key function that calls process_key with the result of pid_label.
    """
    def make_key(x_pid):
        x_label = pid_label(x_pid, pid_names)
        return process_key(x_label)
    return make_key

def process_key(x):
    """
    Compare process names preferring map-agent.
    """

    if x.startswith('map-agent'):
        return f"0_{x}"
    else:
        return f"9_{x}"

def is_interesting_process(unfiltered, pid, row):
    """
    Check if this is a graph that we should graph.

    Arguments:
        unfiltered(boolean): if true, return all processes
        pid(int): the process ID parsed as an int from 'row'
        row(dict): dictionary from csv.DictReader
    Returns:
        boolean: true if the process is "interesting"
    """
    if unfiltered:
        return True
    elif pid == system_stats_pid:
        # totals
        return True
    elif row['user'] == 'map':
        # map service
        return True
    elif re.search(r'iftop', row['cmdline']):
        return True
    elif row['name'] == 'fprobe':
        return True
    elif row['name'] == 'java' and re.search(r'map-dns', row['cmdline']):
        # DNS server
        return True
    elif row['docker'] == 'True':
        # docker container or docker system process
        return True
    else:
        return False

def organize_node_data(unfiltered, stats_dir):
    """
    Returned timestamps are unix epoch seconds.

    Arguments:
        unfiltered(boolean): if true, return all processes
        stats_dir(Path): base directory to read stats data from
    Returns:
        data_cpu(dict): ts -> pid -> cpu
        data_memory(dict): ts -> pid -> memory
        all_pids(set): all process ids
        pid_names(dict): pid -> display name
        network_data(dict): attr -> interface -> ts -> value
    """
    # timestamps are unix epoch seconds
    data_cpu = dict() # ts -> pid -> cpu
    data_memory = dict() # ts -> pid -> memory
    all_pids = set()
    pid_names = dict() # pid -> display name

    network_data = dict() # attr -> interface -> ts -> value

    for file in stats_dir.iterdir():
        match = re.match(r'^(\d+)\.csv$', file.name)
        net_match = re.match(r'^(\d+)_net\.csv$', file.name)
        if file.is_file() and match:
            timestamp = int(match.group(1))
            ts_data_cpu = dict() # pid -> cpu
            ts_data_memory = dict() # pid -> memory
            with open(file) as f:
                reader = csv.DictReader(f)
                for row in reader:
                    pid = int(row['pid'])
                    if is_interesting_process(unfiltered, pid, row):
                        cpu = float(row['cpu_user'])
                        # vms is not useful, it includes things such as file backed memory
                        memory = float(row['memory_rss'])
                        if pid != system_stats_pid:
                            cpu = cpu + float(row['cpu_sys'])

                            if row['name'] == 'java':
                                match = re.search(r'([^/\s]+)\.jar', row['cmdline'])
                                if match:
                                    name = match.group(1)
                                else:
                                    name = row['name']
                            else:
                                name = row['name']
                            pid_names[pid] = name

                        ts_data_cpu[pid] = cpu
                        ts_data_memory[pid] = memory
                        if pid != system_stats_pid:
                            all_pids.add(pid)

            # verify that we have a system entry
            if system_stats_pid not in ts_data_cpu:
                get_logger().warning("Missing system entry in %s, skipping", file)
                continue

            data_cpu[timestamp] = ts_data_cpu
            data_memory[timestamp] = ts_data_memory

        elif net_match:
            timestamp = int(net_match.group(1))
            with open(file) as f:
                reader = csv.DictReader(f)
                for row in reader:
                    interface = row['interface']
                    interface = interface.lower()
                    for attr, value in row.items():
                        if attr == 'interface':
                            continue
                        attr_data = network_data.get(attr, dict())
                        interface_data = attr_data.get(interface, dict())

                        try:
                            interface_data[timestamp] = int(value)
                        except ValueError:
                            # missing column
                            interface_data[timestamp] = 0

                        attr_data[interface] = interface_data
                        network_data[attr] = attr_data

    return data_cpu, data_memory, all_pids, pid_names, network_data

def compute_node_differential_percent(all_pids, data):
    """Compute percentage of the system used at each timestamp as a
    difference from the previous timestamp.  This assumes that the
    data measured is a counter that increases over time and the
    difference between 2 timestamps is the useful information.

    """
    data_percent = dict() # timestamp -> pid -> data

    prev_values = None
    for timestamp, ts_data in sorted(data.items()):
        ts_percent = dict() # pid -> data
        if prev_values is not None:
            system_cur = float(ts_data[system_stats_pid])
            system_prev = (prev_values[system_stats_pid])
            system_diff = system_cur - system_prev

            for pid in all_pids:
                cur = ts_data.get(pid, None)
                prev = prev_values.get(pid, None)
                if cur is None or prev is None:
                    percent = 0
                else:
                    diff = float(cur - prev)
                    if diff < 0:
                        get_logger().warn("Found negative usage difference at %d for %d diff %f", timestamp, pid, diff)
                        percent = 0
                    else:
                        percent = diff / system_diff
                        if percent > 1:
                            get_logger().warn("Found diff (%f) that is greater than system diff (%f)", diff, system_diff)

                ts_percent[pid] = percent * 100

        data_percent[timestamp] = ts_percent
        prev_values = ts_data

    return data_percent

def compute_node_absolute_percent(all_pids, data):
    """Compute percentage of the system that is used at each timestamp.
    This assumes that the values at each timestamp are absolute
    measurements.

    """
    data_percent = dict() # timestamp -> pid -> data

    for timestamp, ts_data in sorted(data.items()):
        ts_percent = dict() # pid -> data
        for pid in all_pids:
            system_cur = float(ts_data[system_stats_pid])
            cur = ts_data.get(pid, None)
            if cur is None:
                percent = 0
            else:
                percent = float(cur) / system_cur
            ts_percent[pid] = percent * 100
        data_percent[timestamp] = ts_percent

    return data_percent

def write_percents(all_pids, pid_names, filename, data_percent):
    with open(filename, 'w') as f:
        writer = csv.writer(f)
        header = ['timestamp']
        for pid in sorted(all_pids):
            label = pid_label(pid, pid_names)
            header.append(label)
        writer.writerow(header)

        for timestamp, ts_percent in sorted(data_percent.items()):
            row = [timestamp]
            for pid in sorted(all_pids):
                if pid in ts_percent:
                    row.append(ts_percent[pid])
                else:
                    row.append(0)
            writer.writerow(row)

def write_cpu_plot(node_name, file_cpu, output, first_timestamp_sec):
    get_logger().debug("Writing plot for %s with first timestamp of %d from %s", node_name, first_timestamp_sec, file_cpu)

    df = pd.read_csv(file_cpu)
    df['time_minutes'] = (df['timestamp'] - first_timestamp_sec) / 60
    max_minutes = df['time_minutes'].max()

    fig, ax = map_utils.subplots()
    ax.set_title(f"{node_name} Process CPU load")
    ax.set_ylabel('% CPU')
    ax.set_xlabel('time (minutes)')
    ax.set_xlim(left=0, right=max_minutes)

    ydata = list()
    labels = list()
    for process in sorted(df.columns, key=process_key):
        if process != 'timestamp' and process != 'time_minutes':
            ydata.append(df[process])
            labels.append(process)

    ax.stackplot(df['time_minutes'], ydata, labels=labels)
    handles, labels = ax.get_legend_handles_labels()
    lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left")

    output_name = output / f"{file_cpu.stem}.png"
    fig.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd,), bbox_inches='tight')
    plt.close(fig)

def write_memory_plot(node_name, file_memory, output, first_timestamp_sec):
    df = pd.read_csv(file_memory)
    df['time_minutes'] = (df['timestamp'] - first_timestamp_sec) / 60
    max_minutes = df['time_minutes'].max()

    fig, ax = map_utils.subplots()
    ax.set_title(f"{node_name} Process memory load")
    ax.set_ylabel('% memory')
    ax.set_xlabel('time (minutes)')
    ax.set_xlim(left=0, right=max_minutes)

    ydata = list()
    labels = list()
    for process in sorted(df.columns, key=process_key):
        if process != 'timestamp' and process != 'time_minutes':
            ydata.append(df[process])
            labels.append(process)

    ax.stackplot(df['time_minutes'], ydata, labels=labels)

    handles, labels = ax.get_legend_handles_labels()
    lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left")

    output_name = output / f"{file_memory.stem}.png"
    fig.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd,), bbox_inches='tight')
    plt.close(fig)

def write_memory_absolute_plot(node_name, data_memory, all_pids, pid_names, output, first_timestamp_sec):
    """
    Write out the graph of absolute memory.

    Arguments:
        node_name(str): name of the node
        data_memory(dict): ts(int) -> pid(int) -> memory(float)
        all_pids(set): all process names
        pid_names(dict): pid(int) -> pid name(str)
        output(Path): where the graphs are being written
    """

    timestamps = sorted(data_memory.keys())
    minutes = [ (ts - first_timestamp_sec) / 60 for ts in timestamps ]
    max_minutes = max(minutes)

    fig, ax = map_utils.subplots()
    ax.set_title(f"{node_name} Process memory")
    ax.set_ylabel('GB memory')
    ax.set_xlabel('time (minutes)')
    ax.set_xlim(left=0, right=max_minutes)

    ydata = list()
    labels = list()
    for pid in sorted(all_pids, key=make_pid_key(pid_names)):
        plot_data = list()
        for timestamp in timestamps:
            ts_data = data_memory[timestamp]
            if pid in ts_data:
                memory = bytes_to_gigabytes(ts_data[pid])
            else:
                memory = 0
            plot_data.append(memory)
        label = pid_label(pid, pid_names)
        ydata.append(plot_data)
        labels.append(label)

    ax.stackplot(minutes, ydata, labels=labels)
    handles, labels = ax.get_legend_handles_labels()
    lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left")

    output_name = output / f"{node_name}_stats_memory_absolute.png"
    fig.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd,), bbox_inches='tight')
    plt.close(fig)

def compute_network_differentials(network_data):
    """
    Arguments:
        network_data(dict): attr -> interface -> ts -> value
    Returns:
        network_data(dict): attr -> interface -> ts -> diff from previous per second
    """
    retval = dict()

    for attr, attr_data in network_data.items():
        ret_attr_data = retval.get(attr, dict())

        for interface, interface_data in attr_data.items():
            ret_interface_data = ret_attr_data.get(interface, dict())

            prev_ts = None
            prev_value = None
            for ts, value in sorted(interface_data.items()):
                if prev_value is not None:
                    diff_value = value - prev_value
                    diff_ts = ts - prev_ts
                    diff_per_second = diff_value / diff_ts
                    ret_interface_data[ts] = diff_per_second
                prev_value = value
                prev_ts = ts

            ret_attr_data[interface] = ret_interface_data
        retval[attr] = ret_attr_data
    return retval

def write_network_bandwidth_plot(output_dir, first_timestamp_sec, node_name, graph_label, file_label, bandwidth_data, ip_info):
    """
    Arguments:
        output_dir(Path): base directory for output
        label(str): label for the data
        bandwidth_data(dict): interface -> ts -> value
        first_timestamp_sec(int): the first timestamp for relative computation
        ip_info(dict): node -> ifce -> ip
    """
    node_ip_info = ip_info.get(node_name, dict())
    get_logger().debug("Got ip info for %s -> %s", node_name, node_ip_info)

    fig, ax = map_utils.subplots()
    ax.set_title(graph_label)
    ax.set_ylabel('mbps')
    ax.set_xlabel('time (minutes)')

    max_minutes=0
    for interface, interface_data in bandwidth_data.items():
        xdata = list()
        ydata = list()
        for ts, value in interface_data.items():
            rel_sec = ts - first_timestamp_sec
            rel_min = rel_sec / 60
            mbps = value / 1024 / 1024 * 8 # 1024 kb / byte, 1024 kb / mb, 8 bits per byte
            xdata.append(rel_min)
            ydata.append(mbps)
            max_minutes = max(max_minutes, rel_min)

        if interface in node_ip_info:
            ip = f" - {node_ip_info[interface]}"
        else:
            ip = ""
        ax.plot(xdata, ydata, label=f"{interface}{ip}")

    ax.set_xlim(left=0, right=max_minutes)

    handles, labels = ax.get_legend_handles_labels()
    lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left")

    output_name = output_dir / f"{node_name}_stats_network-{file_label}.png"
    fig.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd,), bbox_inches='tight')
    plt.close(fig)

def gather_network_ips(sim_output):
    """
    Determine the IP address for each interface on each node.
    If the information isn't available, then the result will be an empty dictionary.
    Use of the dictionary should always check for the information existing

    Arguments:
        sim_output(Path): directory that the sim output is in

    Returns:
        dict: node_name -> ifce_name -> ip
    """

    network_info = sim_output / "inputs/network-information.txt"
    if not network_info.exists():
        # no information
        return dict()

    info = dict()
    node_name = None
    ifce_name = None
    node_name_re = re.compile(r'^(\S+)\s+\|\s+CHANGED')
    ifce_name_re = re.compile(r'^\d+:\s+([^@]+)(@\S+)?: \<')
    ip_re = re.compile(r'^\s+inet ([0-9\.]+)/')
    with open(network_info) as f:
        for line in f:
            node_name_match = node_name_re.match(line)
            if node_name_match:
                node_name = node_name_match.group(1).split('.')[0]
                ifce_name = None
                get_logger().debug("Found node %s", node_name)
                continue

            ifce_name_match = ifce_name_re.match(line)
            if ifce_name_match:
                ifce_name = ifce_name_match.group(1)
                get_logger().debug("Found interface %s", ifce_name)
                continue

            ip_match = ip_re.match(line)
            if ip_match:
                ip = ip_match.group(1)
                if node_name is not None and ifce_name is not None:
                    node_name = node_name.lower()
                    ifce_name = ifce_name.lower()
                    get_logger().debug("Found ip %s on %s", ip, node_name)
                    node_info = info.get(node_name, dict())
                    node_info[ifce_name] = ip
                    info[node_name] = node_info
                else:
                    get_logger().warning("Found network interface name %s and node_name (%s) or ifce_name (%s) is missing", ip, node_name, ifce_name)

                ifce_name = None

    return info

def process_node_dir(unfiltered, output, node_dir, first_timestamp_sec, ip_info):
    try:
        stats_dir = node_dir / 'system_stats'
        if not stats_dir.exists():
            # not a directory we care about
            return

        node_name = node_dir.stem.lower()
        get_logger().info("Processing node %s", node_name)

        (data_cpu, data_memory, all_pids, pid_names, network_data) = organize_node_data(unfiltered, stats_dir)
        percent_cpu = compute_node_differential_percent(all_pids, data_cpu)
        percent_memory = compute_node_absolute_percent(all_pids, data_memory)

        file_cpu = output / f"{node_name}_stats_cpu.csv"
        file_memory = output / f"{node_name}_stats_memory.csv"
        write_percents(all_pids, pid_names, file_cpu, percent_cpu)
        write_percents(all_pids, pid_names, file_memory, percent_memory)
        write_cpu_plot(node_name, file_cpu, output, first_timestamp_sec)
        write_memory_plot(node_name, file_memory, output, first_timestamp_sec)

        write_memory_absolute_plot(node_name, data_memory, all_pids, pid_names, output, first_timestamp_sec)

        network_per_second = compute_network_differentials(network_data)

        if 'rx_bytes' in network_per_second:
            write_network_bandwidth_plot(output, first_timestamp_sec, node_name, 'RX mbps', 'rx', network_per_second['rx_bytes'], ip_info)

        if 'tx_bytes' in network_per_second:
            write_network_bandwidth_plot(output, first_timestamp_sec, node_name, 'TX mbps', 'tx', network_per_second['tx_bytes'], ip_info)

    except:
        get_logger().exception("Unexpected error")

def main_method(args):
    sim_output = Path(args.sim_output)
    if not sim_output.exists():
        get_logger().error("%s does not exist", sim_output)
        return 1

    with open(args.first_timestamp_file) as f:
        ts_str = f.readline().strip()
        first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
    get_logger().info("Simulation started at %s", first_timestamp)

    ip_info = gather_network_ips(sim_output)
    get_logger().debug("ip_info: %s", ip_info)

    first_timestamp_secs = first_timestamp.timestamp()

    output = Path(args.output)
    output.mkdir(parents=True, exist_ok=True)

    with multiprocessing.Pool(processes=os.cpu_count()) as pool:
        results = list()

        for node_dir in sim_output.iterdir():
            if node_dir.is_dir():
                results.append(pool.apply_async(func=process_node_dir, args=[args.unfiltered, output, node_dir, first_timestamp_secs, ip_info]))

        for result in results:
            result.wait()

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    class ArgumentParserWithDefaults(argparse.ArgumentParser):
        '''
        From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
        '''
        def add_argument(self, *args, help=None, default=None, **kwargs):
            if help is not None:
                kwargs['help'] = help
            if default is not None and args[0] != '-h':
                kwargs['default'] = default
                if help is not None:
                    kwargs['help'] += ' (default: {})'.format(default)
            super().add_argument(*args, **kwargs)

    parser = ArgumentParserWithDefaults(formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json')
    parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true')
    parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True)
    parser.add_argument("-s", "--sim-output", dest="sim_output", help="Chart output directory (Required)", required=True)
    parser.add_argument("--first-timestamp-file", dest="first_timestamp_file", help="Path to file containing the log timestamp that the simulation started", required=True)
    parser.add_argument("--unfiltered", dest="unfiltered", help="Show all processes, not just MAP processes", action='store_true')

    args = parser.parse_args(argv)

    map_utils.setup_logging(default_path=args.logconfig)
    if 'multiprocessing' in sys.modules:
        import multiprocessing_logging
        multiprocessing_logging.install_mp_handler()

    if args.debug:
        import pdb, traceback
        try:
            return main_method(args)
        except:
            extype, value, tb = sys.exc_info()
            traceback.print_exc()
            pdb.post_mortem(tb)
    else:
        return main_method(args)

if __name__ == "__main__":
    sys.exit(main())


import web
# the urls parts makes the link between "/" and the "index" class
urls = (
"/","index"
)

app = web.application(urls,globals())

render = web.template.render('templates/')

class index:
# GET(self) function is a default function to look for event handle when using web.py
    def GET(self):
        greeting = "Waaagahhhh!"
        # the (greeting = greeting) is given the local greeting var in this .py script to the greeting argument in the template
        # when change the line to render.index(greeting = None),the webpage would display the default message
        return render.index(greeting = greeting)

if __name__ == "__main__":

    app.run()


import numpy as np
import matplotlib.pyplot as plt

def f1(x):
   f1 = x**4.0
   return f1;

# trapezoidal rule
def trapezoid(f,a,b,N):
    h   = (b-a)/N
    xi  = np.linspace(a,b,N+1)
    fi  = f(xi)
    s   = 0.0
    for i in range(1,N):
        s = s + fi[i]
    s = (h/2)*(fi[0] + fi[N]) + h*s
    return s

# romberg method
def romberg(f,a,b,eps,nmax):
# f     ... function to be integrated
# [a,b] ... integration interval
# eps   ... desired accuracy
# nmax  ... maximal order of Romberg method
    Q  = np.zeros((nmax,nmax),float)
    converged = 0
    for i in range(0,nmax):
        N      = 2**i
        Q[i,0] = trapezoid(f,a,b,N)
        for k in range(0,i):
            n        = k + 2
            Q[i,k+1] = 1.0/(4**(n-1)-1)*(4**(n-1)*Q[i,k] - Q[i-1,k])
        if (i > 0):
            if (abs(Q[i,k+1] - Q[i,k]) < eps):
               converged = 1
               break
    print(Q[i,k+1],N,converged)
    return (Q[i,k+1],N,converged)

# main program
a  = 0.0;b = 1.0  # integration interval [a,b]
romberg(f1,a,b,1.0e-12,10)


input_string = input("Enter a list of positive integers separated by a space:\n")
number_list  = input_string.split()
origin_a = 0
origin_b = 1
memory = 0
counter = 0

for x in number_list:

    while origin_b <= abs(int(x)):
        memory = origin_b
        origin_b = origin_a + origin_b
        origin_a = memory

    if origin_a == abs(int(x)):
        counter = counter + 1
    else:
        break

if counter == len(number_list):
    print("This set of numbers belongs to Fibonacci Series.")
else:
    print("This set of numbers does not belong to Fibonacci Series.")


# imports
from couchbase.bucket import Bucket
from couchbase.n1ql import N1QLQuery
import pandas as pd

# helper function to flatten the json for data frame construction
def flatten_json(y):
    out = {}

    def flatten(x, name=''):
        if type(x) is dict:
            for a in x:
                flatten(x[a], name + a + '_')
        elif type(x) is list:
            i = 0
            for a in x:
                flatten(a, name + str(i) + '_')
                i += 1
        else:
            out[name[:-1]] = x

    flatten(y)
    return out

# get connection
bkt = Bucket('http://54.235.239.211:8091/default')

# run query
query = N1QLQuery("SELECT * FROM default WHERE ImageGuid IS NOT NULL")
rows = bkt.n1ql_query(query)

# create data frame for each result pulled from query
df_appended = None
for row in rows:
    result = row['default']
    # create data frames for items unique to result
    image_guid = flatten_json(result['ImageGuid'])
    guid_df = pd.io.json.json_normalize(image_guid)
    guid_df.columns = ['Guid']
    results_key = flatten_json(result['ResultsKey'])
    key_df = pd.io.json.json_normalize(results_key)
    key_df.columns = ['ResultsKey']
    # iterate over faces in result
    for i in range(len(result['EmotionResults'])):
        emotion_results = flatten_json(result['EmotionResults'][i])
        emotion_df = pd.io.json.json_normalize(emotion_results)
        face_results = flatten_json(result['FaceResults'][i])
        face_df = pd.io.json.json_normalize(face_results)
        face_df = face_df.drop(['FaceRectangle_Left', 'FaceRectangle_Top', 'FaceRectangle_Height', 'FaceRectangle_Width'], axis = 1)
        df_row = pd.concat([guid_df, key_df, emotion_df, face_df], axis = 1)
        # concatenate all 4 data frames
        if df_appended is None:
            df_appended = df_row
        else:
            df_appended = pd.concat([df_appended, df_row], axis = 0)