#!/usr/bin/env python
# coding: utf-8

# In[9]:

import numpy as np
import matplotlib.pyplot as plt

#读取数据
def LoadDataSet(filename):
    DataMatrix = []
    LabelMatrix = []
    fr = open(filename)
    for line in fr.readlines():
        LineArray = line.strip().split('\t')         #逐行读取，滤除空格等
        DataMatrix.append([float(LineArray[0]), float(LineArray[1])])#添加数据
        LabelMatrix.append(float(LineArray[2]))                    #添加标签
    return DataMatrix, LabelMatrix

def SelectJrand(i, m): # 随机选择alpha
    j = i
    while j == i:
        j = int(np.random.uniform(0,m))
    return j

def ClipAlpha(alpha, H, L):
    #限幅，alpha的上下界
    if alpha > H:
        alpha = H
    if alpha < L:
        alpha = L
    return alpha

def SMOSimple(datamatrix, labelmatrix, C, tolerant, maxiter):#数据矩阵、标签矩阵、松弛变量、容错率、最大迭代次数
    DataMatrix = np.mat(datamatrix)
    LabelMatrix = np.mat(labelmatrix).transpose()
    b = 0
    m,n = np.shape(DataMatrix)
    Alphas = np.mat(np.zeros((m,1)))
    Iter = 0
    #最多迭代maxiter次
    while Iter < maxiter:
        AlphaPairsChanged = 0
        for i in range(m):
            #步骤1：计算误差Ei
            fXi = float(np.multiply(Alphas,LabelMatrix).T*(DataMatrix*DataMatrix[i,:].T)) + b #预测值
            Ei = fXi - float(LabelMatrix[i]) #误差
            #优化alpha，更设定一定的容错率
            #不满足KKT条件，更新的第一个条件
            if ((LabelMatrix[i]*Ei < -tolerant) and (Alphas[i] < C)) or ((LabelMatrix[i]*Ei > tolerant) and (Alphas[i] > 0)):
                #随机选择另一个与alpha_i成对优化的alpha_j，按道理max|e1-e2|
                j = SelectJrand(i,m)
                #步骤1：计算误差Ej
                fXj = float(np.multiply(Alphas,LabelMatrix).T*(DataMatrix*DataMatrix[j,:].T)) + b
                Ej = fXj - float(LabelMatrix[j])
                #保存更新前的aplpha值，使用深拷贝
                AlphasIold = Alphas[i].copy()
                AlphasJold = Alphas[j].copy()
                #步骤2：计算上下界L和H
                if LabelMatrix[i] != LabelMatrix[j]:      #y1≠y2
                    L = max(0, Alphas[j] - Alphas[i])
                    H = min(C, C + Alphas[j] - Alphas[i])
                else:
                    L = max(0, Alphas[j] + Alphas[i] - C)  #y1=y2
                    H = min(C, Alphas[j] + Alphas[i])

                #步骤3：计算eta，2*k12-k11-k22
                Eta = 2.0 * DataMatrix[i,:]*DataMatrix[j,:].T - DataMatrix[i,:]*DataMatrix[i,:].T - DataMatrix[j,:]*DataMatrix[j,:].T
                if Eta >= 0:
                    print('Eta>=0')
                    continue
                #步骤4：更新alpha_j
                Alphas[j] -= LabelMatrix[j]*(Ei - Ej)/Eta #alpha2更新公式
                #步骤5：修剪alpha_j
                Alphas[j] = ClipAlpha(Alphas[j], H, L) #更新alpha2
#                 if abs(Alphas[j]-AlphasJold) < 0.00001:
#                     print('alpha_j变化太小')
#                     continue
                #步骤6：更新alpha_i ，alpha1+y1y2(alpha2old-alpha2new)
                Alphas[i] += LabelMatrix[j]*LabelMatrix[i]*(AlphasJold - Alphas[j])
                #步骤7：更新b_1和b_2
                b1 = b - Ei - LabelMatrix[i]*(Alphas[i] - AlphasIold)*DataMatrix[i,:]*DataMatrix[i,:].T - LabelMatrix[j]*(Alphas[j]-AlphasJold)*DataMatrix[i,:]*DataMatrix[j,:].T
                b2 = b - Ej - LabelMatrix[i]*(Alphas[i] - AlphasIold)*DataMatrix[i,:]*DataMatrix[j,:].T - LabelMatrix[j]*(Alphas[j]-AlphasJold)*DataMatrix[j,:]*DataMatrix[j,:].T
                #步骤8：根据b_1和b_2更新b
                if (0 < Alphas[i]) and (C > Alphas[i]):   #  0<alpha1<c
                    b = b1
                elif (0 < Alphas[j]) and (C > Alphas[j]): #  0<alpha2<c
                    b = b2
                else:
                    b = (b1 + b2)/2.0      # b_new
                #统计优化次数
                AlphaPairsChanged += 1

        #更新迭代次数
        if AlphaPairsChanged == 0:
            Iter += 1
        else:
            Iter = 0
#         print("迭代次数: %d" % Iter)
    return b, Alphas

def ShowClassifer(dataMat, labelMat, w, b):
    #绘制样本点
    data_plus = []                                  #正样本
    data_minus = []                                 #负样本
    for i in range(len(dataMat)):
        if labelMat[i] > 0:
            data_plus.append(dataMat[i])
        else:
            data_minus.append(dataMat[i])
    data_plus_np = np.array(data_plus)              #转换为numpy矩阵
    data_minus_np = np.array(data_minus)            #转换为numpy矩阵
    plt.scatter(np.transpose(data_plus_np)[0], np.transpose(data_plus_np)[1], c='green')   #正样本散点图
    plt.scatter(np.transpose(data_minus_np)[0], np.transpose(data_minus_np)[1], c='blue') #负样本散点图

#     x1=np.linspace(1,10,10) # 取点
#     print('x1=',x1)
#     x2=(-b-w[0]*x1)/w[1] #拟合方程
#     print('x2=',x2)
#     plt.plot(x1,x2 , color = 'red')
   #绘制直线
    x1 = max(dataMat)[0] - 0.35*(max(dataMat)[0]-min(dataMat)[0])
    x2 = min(dataMat)[0] + 0.35*(max(dataMat)[0]-min(dataMat)[0])
    a1, a2 = w
    b = float(b)
    a1 = float(a1[0])
    a2 = float(a2[0])
    y1, y2 = (-b- a1*x1)/a2, (-b - a1*x2)/a2
    plt.plot([x1, x2], [y1, y2])
    #找出支持向量点，0< alpha <C
    for i, alpha in enumerate(alphas):
        if alpha > 0:
            x, y = dataMat[i]
            plt.scatter([x], [y], s=150, c='none', alpha=0.7, linewidth=1.5, edgecolor='red')
    plt.show()

def get_w(dataMat, labelMat, alphas):
    alphas, dataMat, labelMat = np.array(alphas), np.array(dataMat), np.array(labelMat)
    w = np.dot((np.tile(labelMat.reshape(1, -1).T, (1, 2)) * dataMat).T, alphas) #np.tile原矩阵横向、纵向地复制,w=∑alpha*xi*yi
    return w.tolist()  # .tolist将数组或者矩阵转换成列表

if __name__ == '__main__':
    dataArr, labelArr = LoadDataSet('testSet1.txt')
    b,alphas = SMOSimple(dataArr, labelArr, 0.6, 0.001, 40)#数据矩阵、标签矩阵、松弛变量、容错率、最大迭代次数
    print('alphas=',alphas)
    w = get_w(dataArr, labelArr, alphas)
    print('w=',w)
    print('b=',b)
    ShowClassifer(dataArr, labelArr, w, b)


import jax
import jax.nn as nn
import jax.numpy as jnp

from nmax import Module
from cvax.modules import Dense, Conv2d, BatchNorm2d, MaxPool2d

class ResStem(Module):

    conv: Module
    bn: Module

    def __init__(self,
        rng,
        kernel_shape: tuple[int, int, int, int],
        stride: int,
        ):
        self.conv = Conv2d(rng, kernel_shape, stride=stride)
        self.bn = BatchNorm2d(kernel_shape[0])
        self.pool = MaxPool2d(kernel_width=3, stride=2, padding='SAME')

    def forward(self, x):
        x = self.conv(x)
        x = self.bn(x)
        x = nn.relu(x)
        x = self.pool(x)
        return x

class ResHead(Module):

    dense: Module

    def __init__(self, rng, in_channels, out_dim):
        self.dense = Dense(rng, in_channels, out_dim)

    def forward(self, x):
        x = jnp.mean(x, axis=(2, 3))
        x = self.dense(x)
        return x

class BottleneckBlock(Module):

    conv_proj: Module
    bn_proj: Module

    conv_in: Module
    bn_in: Module

    conv_bt: Module
    bn_bt: Module

    conv_out: Module
    bn_out: Module

    def __init__(self,
        rng,
        in_channels: int,
        bt_channels: int,
        out_channels: int,
        in_stride: int = 1,
        ):

        rngs = jax.random.split(rng, num=4)

        self.use_proj = (in_channels != out_channels or in_stride != 1)

        if self.use_proj:
            self.conv_proj = Conv2d(rngs[0], (out_channels, in_channels, 1, 1), stride=in_stride)
            self.bn_proj = BatchNorm2d(out_channels)

        self.conv_in = Conv2d(rngs[1], (bt_channels, in_channels, 1, 1), stride=in_stride)
        self.bn_in = BatchNorm2d(bt_channels)

        self.conv_bt = Conv2d(rngs[2], (bt_channels, bt_channels, 3, 3))
        self.bn_bt = BatchNorm2d(bt_channels)

        self.conv_out = Conv2d(rngs[3], (out_channels, bt_channels, 1, 1))
        self.bn_out = BatchNorm2d(out_channels)

    def forward(self, x):
        residual = x
        if self.use_proj:
            residual = self.conv_proj(residual)
            residual = self.bn_proj(residual)

        x = self.conv_in(x)
        x = self.bn_in(x)
        x = nn.relu(x)

        x = self.conv_bt(x)
        x = self.bn_bt(x)
        x = nn.relu(x)

        x = self.conv_out(x)
        x = self.bn_out(x)
        x = nn.relu(x + residual)

        return x


"""Set up one more more logger(s)."""

import time
import logging
from config import settings

# possible log levels
LEVELS = ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG')

# default log file to be used when using the the combined strategy
LOG_FILE = 'reactor.log'

# default log level
LEVEL = 'INFO'

# log strategies that can be used. with combined, all logs go to one file;
# with split, each pikaview has its own log file.
LOG_FILE_STRATEGIES = ('split', 'combined')

# default log strategy
LOG_FILE_STRATEGY_DEFAULT = 'combined'

def get_log_file_strategy():
    return LOG_FILE_STRATEGY_DEFAULT

def get_module_log_level(name):
    """Reads config file for a log level set for this module."""
    try:
        log_level = settings.get('logs').get('level')
        if log_level.upper() in LEVELS:
            return log_level
    except AttributeError:
        # if the logs section doesn't exist, use default
        return LEVEL

    if log_level == "":
        return LEVEL

def get_log_file(name):
    return LOG_FILE

def get_logger(name):
    """
    Returns a properly configured STDERR logger
         name (str) should be the module name.
    """
    FORMAT = "[%(levelname)s] %(asctime)s: %(message)s"
    DATEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
    logging.Formatter.converter = time.gmtime

    logger = logging.getLogger(name)
    level = get_module_log_level(name)
    logger.setLevel(level)

    # handler = logging.FileHandler(get_log_file(name))
    # handler.setFormatter(logging.Formatter(FORMAT, datefmt=DATEFORMAT))
    # handler.setLevel(level)
    # logger.addHandler(handler)

    stderrLogger = logging.StreamHandler()
    stderrLogger.setFormatter(logging.Formatter(FORMAT, datefmt=DATEFORMAT))
    logger.addHandler(stderrLogger)

    return logger


'''
Parhuam Jalalian
CIS41A Spring 2020
Unit D take-home assignment Script
'''
from collections import namedtuple

# Part One Sets
print("Execution results:")

# a. Create a set named class1 and populate it with the students Li, Audry, Jia, Migel, Tanya.
class1 = { 'Li', 'Audry', 'Jia', 'Migel', 'Tanya'}

# b. Create a set named class2 and populate it with the students Sasha, Migel, Tanya, Hiroto, Audry.
class2 = {'Sasha', 'Migel', 'Tanya', 'Hiroto', 'Audry'}

# c. Create a set named class3 and populate it with the students Migel, Zhang, Hiroto, Anita, Jia.
class3 = {'Migel', 'Zhang', 'Hiroto', 'Anita', 'Jia'}

# d. Print a sorted list of students who are in all three classes.
print("Students in all three classes:", sorted(class1.intersection(class2, class3)))

# e. Print a sorted list of all students.
print("All students", sorted(class1.union(class2, class3)))

# f. Print a sorted list of all students in class1 but not class2 or class3.
print("Students in class1 but not class2 or class3:", sorted(class1.difference(class2, class3)))

# Part Two Swap
# a. Create a list containing elements 1, 2, 3
list1 = [1,2,3]

# b. Swap the second and third elements of the list. Do this with one line of code.
list1[2], list1[1] = list1[1] , list1[2]

# c. Print the list.
print("List after swap:", list1)

# Part Three Tuple Basics
# a. Create a tuple that contains the elements Casablanca, 1942, romantic drama.
tuple1 = ('Casablanca', 1942, 'romantic drama')

# b. Unpack the tuple into variables title, year, genre.
title = tuple1[0]
year = tuple1[1]
genre = tuple1[2]

# c. Print the title.
print("My favorite movie is:", title)

# Part Four Named Tuple
# a. Define a named tuple called Movie that contains the fields title, year, genre.
Movie = namedtuple('Movie', 'title year genre')

# b. Create a Movie tuple that contains the elements Casablanca, 1942, romantic drama.
casab = Movie(title = 'Casablanca', year = 1942, genre = 'romantic drama')

# c. Print the title
print("My favorite movie is:", casab.title)

# Part Five Named Tuple Containing a List
# a. Define a named tuple called Moviestars that contains the fields title, year, genre, stars.
Moviestars = namedtuple('Moviestars', 'title year genre stars')

# b. Create a Moviestars tuple called favoritemovie that contains the elements Casablanca, 1942, romantic drama,
#    and a list containing elements Humphrey Bogart, Ingrid Bergman.
list2 = ['Humphrey Bogart', 'Ingrid Bergman']
favoritemovie = Moviestars('Casablanca', 1942, 'romantic drama', list2)

# c. Append Claude Rains to the stars list. Hint: Use the syntax favoritemovie.stars.append
favoritemovie.stars.append('Claude Rains')

# d. Print star Ingrid Bergman.
print("My favorite star is:", favoritemovie[3][1])

# e. Print favoritemovie.
print(favoritemovie)

'''
Execution results:
Students in all three classes: ['Migel']
All students ['Anita', 'Audry', 'Hiroto', 'Jia', 'Li', 'Migel', 'Sasha', 'Tanya', 'Zhang']
Students in class1 but not class2 or class3: ['Li']
List after swap: [1, 3, 2]
My favorite movie is: Casablanca
My favorite movie is: Casablanca
My favorite star is: Ingrid Bergman
Moviestars(title='Casablanca', year=1942, genre='romantic drama', stars=['Humphrey Bogart', 'Ingrid Bergman', 'Claude Rains'])
'''


import os
import sys
if 'exp_commands' not in os.getcwd():
    os.chdir('exp_commands/')
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)
sys.path.append('../py_scripts')
from py_scripts.model_params import *
from py_scripts.dataset_params import *
if 'exp_commands' in os.getcwd():
    os.chdir('..')
########### EXPERIMENTS TO RUN ##############

"""
Getting nice receptive fields for the 10K neuron model. And for ReLU Baselines as a comparison.
"""

settings_for_all = dict(
    epochs_to_train_for = 1000,
    dataset = DataSet.CIFAR10,
    classification=True,
    adversarial_attacks=False,
    validation_neuron_logger = True,
    log_metrics = True,
    log_receptive_fields = True,
    num_binary_activations_for_gaba_switch=20000000,
    nneurons=[10000],
    count_dead_train_neurons = True,
)

name_suffix = "_NiceReceptiveFields_10MSwitch_RawCIFAR_StaleGradients_Longer"

exp_list = [

    dict(
        model_style= ModelStyles.SDM,
        test_name= "SDM_SGD_lr=0.2_PosWeights",
        k_approach = "GABA_SWITCH_ACT_BIN",
        opt="SGD",
        lr=0.1,
        all_positive_weights = True,
    ),

    dict(
        model_style= ModelStyles.SDM,
        test_name= "SDM_SGDM_lr=0.09_PosWeights",
        k_approach = "GABA_SWITCH_ACT_BIN",
        opt="SGDM",
        lr=0.03,
        all_positive_weights = True,
    ),

    dict(
        model_style= ModelStyles.SDM,
        test_name= "SDM_SGD_lr=0.2_NegWeights",
        k_approach = "GABA_SWITCH_ACT_BIN",
        opt="SGD",
        lr=0.1,
        all_positive_weights = False,
    ),

    dict(
        model_style= ModelStyles.SDM,
        test_name= "SDM_SGDM_lr=0.03_PosWeights",
        k_approach = "GABA_SWITCH_ACT_BIN",
        opt="SGDM",
        lr=0.03,
        all_positive_weights = True,
    ),
    dict(
        model_style= ModelStyles.SDM,
        test_name= "SDM_SGDM_lr=0.03_NegWeights",
        k_approach = "GABA_SWITCH_ACT_BIN",
        opt="SGDM",
        lr=0.03,
        all_positive_weights = False,
    ),

    dict(
        model_style= ModelStyles.CLASSIC_FFN,
        test_name= "FFN_SGDM_lr=0.03_NegWeights",
        opt="SGDM",
        lr=0.03,
    ),
    dict(
        model_style= ModelStyles.CLASSIC_FFN,
        test_name= "FFN_Adam_lr=0.001_NegWeights",
        opt="Adam",
        lr=0.001,
    ),

]

if __name__ == '__main__':
    print(len(exp_list))
    sys.exit(0)


import base64

def Decrypt(key, encryption):
    message = base64.urlsafe_b64decode(encryption).decode()
    dec = []
    for i in range(len(message)):
        key_c = key[i % len(key)]
        dec.append(chr((256 + ord(message[i]) - ord(key_c)) % 256))

    # print('your massage: \n'+ ''.join(dec))
    msg = ''.join(dec)
    return msg


# -*- coding: utf-8 -*-

import os
import time
import datetime as datetime
import pandas as pd

from labonneboite.importer import util as import_util
from labonneboite.importer import settings as importer_settings
from labonneboite.importer.jobs.common import logger
from labonneboite.scripts.impact_retour_emploi.settings import DEBUG
from labonneboite.importer.models.computing import LogsActivityDPAEClean
from labonneboite.common.env import get_current_env, ENV_DEVELOPMENT

# Pandas utils functions
# ----------------------
# function used to create a new column from dateheure column in dpae

def get_date(row):
    return row['kd_dateembauche'][:10]

class JoinActivityLogsDPAE:

    def __init__(self):
        self.dpae_folder_path = importer_settings.INPUT_SOURCE_FOLDER+'/'

    def set_df_activity(self):
        self.df_activity = self.get_activity_logs()

    def get_activity_logs(self):
        logger.info("Loading logs activities")
        engine = import_util.create_sqlalchemy_engine()

        query = "select * from logs_activity order by id"
        if DEBUG:
            query += " ORDER BY RAND() LIMIT 10000"
        df_activity = pd.read_sql_query(query, engine)

        engine.close()

        # TODO : Define how long we should use logs
        # https://valodata.slack.com/archives/C0QR8RYL8/p1562319224015200

        # TODO : Uncomment these lines after the first initialization of the project
        # one_year_date = datetime.date.today() - datetime.timedelta(365)
        # df_activity['dateheure'] = pd.to_datetime(df_activity['dateheure'])
        # df_activity = df_activity[df_activity.dateheure > one_year_date]

        logger.info('Activities logs are loaded')

        return df_activity.astype(str)

    def get_logs_activities_by_sirets(self, sirets):
        if len(sirets) > 1:
            sub_query = "in %s" % str(tuple(sirets))
        else:
            sub_query = "= %s" % sirets[0]
        engine = import_util.create_sqlalchemy_engine()
        query = "select * from logs_activity where siret %s order by dateheure asc;" % sub_query
        df_activity = pd.read_sql_query(query, engine)
        engine.close()
        return df_activity.astype(str)

    def set_most_recent_dpae_file(self):
        self.most_recent_dpae_file = self.get_most_recent_dpae_file()
        # FIXME : Remove this line to pass a specific dpae file for initialisation, we'll have to parse older DPAE files
        # self.most_recent_dpae_file = "lbb_xdpdpae_delta_201908102200.bz2"

    def get_most_recent_dpae_file(self):
        dpae_paths = os.listdir(self.dpae_folder_path)
        # IMPORTANT : Need to copy the DPAE file from /mnt/datalakepe/ to /srv/lbb/data
        # It is certainly ok, because of the importer which also needs this file
        dpae_paths = [i for i in dpae_paths if i.startswith('lbb_xdpdpae_delta')]

        dpae_paths.sort()
        most_recent_dpae_file = dpae_paths[-1]
        if get_current_env() == ENV_DEVELOPMENT:
            most_recent_dpae_file = 'lbb_xdpdpae_delta_201511102200.csv'

        logger.info(f"the DPAE file which will be used is : {most_recent_dpae_file}")

        return most_recent_dpae_file

    def set_last_recorded_hiring_date(self):
        self.date_last_recorded_hiring = self.get_last_recorded_hiring_date()

    def get_last_recorded_hiring_date(self):

        # We want to check if there are already data  in the final table
        table_name_act_dpae = LogsActivityDPAEClean.__tablename__
        query = f"SELECT COUNT(*) FROM {table_name_act_dpae}"
        engine = import_util.create_sqlalchemy_engine()

        # If data in table
        if engine.execute(query).fetchone()[0] > 0:
            query = f"select date_embauche from {table_name_act_dpae} order by date_embauche DESC LIMIT 1"
            row = engine.execute(query).fetchone()
            date_last_recorded_hiring = row[0].split()[0]
        # Else no data
        else:
            # We set the date to the first activity log that has ever been created on LBB
            date_last_recorded_hiring = "2018-08-31"

        engine.close()
        logger.info(f"the most recent date found is {date_last_recorded_hiring}")

        return date_last_recorded_hiring

    def get_compression_to_use_by_pandas_according_to_file(self, dpae_path):
        dpae_file_extension = dpae_path.split('.')[-1]
        if dpae_file_extension == 'csv':
            compression = 'infer'
        else:
            compression = 'bz2'

        return compression

    def get_path_to_saved_csv(self):
        timestr = time.strftime("%Y-%m-%d")
        path_to_csv = f"{self.dpae_folder_path}act_dpae-{timestr}.csv"

        return path_to_csv

    def save_csv_file(self, df_dpae_act, iteration):
        # save file after each chunk of the dpae file used
        file_exists = os.path.isfile(self.path_to_csv)

        # We want to rewrite the CSV file after each new execution of Join activity Logs DPAE extraction
        if iteration == 0:  # First iteration
            if file_exists:
                os.remove(self.path_to_csv)
            df_dpae_act.to_csv(self.path_to_csv, encoding='utf-8', sep='|')
        else:
            with open(self.path_to_csv, 'a') as f:
                df_dpae_act.to_csv(f, header=False, sep='|')

    def join_dpae_activity_logs(self):

        # We use the datas in csv using smaller chunks
        if DEBUG:
            chunksize = 10 ** 2
        else:
            chunksize = 10000

        i = 0
        column_names = ['kc_siret', 'dc_naf_id', 'dc_adresse', 'dc_codepostal', '_',
                        'dn_tailleetablissement', 'dc_communenaissancepays', 'kd_dateembauche',
                        'dc_typecontrat_id', 'dd_datefincdd', 'dc_romev3_1_id', 'dc_romev3_2_id',
                        'kd_datecreation', 'dc_privepublic', 'dc_commune_id', 'dc_natureemploi_id',
                        'dc_qualitesalarie_id', 'dn_dureetravailhebdo', 'dn_dureetravailmensuelle',
                        'dn_dureetravailannuelle', 'nbrjourtravaille', 'iiann', 'dc_lblprioritede',
                        'kn_trancheage', 'duree_pec', 'dc_ididentiteexterne', 'premiere_embauche']

        total_dpae_rows_used = 0
        logger.info(f"CHUNSIZE DF CSV DPAE: {chunksize}")
        self.path_to_csv = self.get_path_to_saved_csv()
        dpae_file = self.most_recent_dpae_file
        logger.info(f"PROCESSING FILE: {dpae_file}")
        self.set_last_recorded_hiring_date()
        for df_dpae in pd.read_csv(self.dpae_folder_path + dpae_file,
                                   header=None,
                                   compression=self.get_compression_to_use_by_pandas_according_to_file(dpae_file),
                                   names=column_names,
                                   sep='|',
                                   index_col=False,
                                   chunksize=chunksize):

            nb_rows = df_dpae.shape[0]
            logger.info(f"Sample of DPAE has : {nb_rows} rows")
            # We keep rows in the DPAE file that has a date > to the date_last_recorded_hiring
            # The dpae used must be after the last recorded emabuche date
            df_dpae['kd_dateembauche_bis'] = df_dpae.apply(lambda row: get_date(row), axis=1)
            df_dpae = df_dpae[df_dpae.kd_dateembauche_bis > self.date_last_recorded_hiring]
            if len(list(df_dpae.kc_siret.unique())) > 0:
                df_activity = self.get_logs_activities_by_sirets(list(df_dpae.kc_siret.unique()))
                nb_rows = df_dpae.shape[0]

                logger.info(f"Sample of DPAE minus old dates has : {nb_rows} rows")

                # convert df dpae columns to 'object'
                df_dpae = df_dpae.astype(str)

                # We join the dpae and activity logs dataframe
                df_dpae_act = pd.merge(
                    df_dpae,
                    df_activity,
                    how='left',
                    left_on=['dc_ididentiteexterne', 'kc_siret'],
                    right_on=['idutilisateur_peconnect', 'siret']
                )

                nb_rows = df_dpae_act.shape[0]
                logger.info(f"Sample of merged activity/DPAE has : {nb_rows} rows")

                # filter on the fact that dateheure activity must be inferior to kd_dateembauche
                df_dpae_act = df_dpae_act[df_dpae_act.kd_dateembauche_bis > df_dpae_act.dateheure]
                df_dpae_act = df_dpae_act.drop(['kd_dateembauche_bis'], axis=1)
                nb_rows = df_dpae_act.shape[0]
                logger.info(f"Sample of merged activity/DPAE with the good dates has : {nb_rows} rows")

                df_dpae_act = df_dpae_act.astype(str)

                # We save the lines of the join we want to keep
                self.save_csv_file(df_dpae_act, i)

                nb_rows = df_dpae_act.shape[0]
                logger.info(f" ==> Nb rows we keep in this sample : {nb_rows} rows")

                total_dpae_rows_used += df_dpae_act.shape[0]
                logger.info(f" ==> Nb total rows that have been kept : {total_dpae_rows_used} rows")
            else:
                logger.info(f"Sample of merged activity/DPAE with the good dates has : 0 rows")
                logger.info(f" ==> Nb rows we keep in this sample : 0 rows")
                logger.info(f" ==> Nb total rows that have been kept : 0 rows")

            nb_total_rows_dpae_used = (i+1) * chunksize
            logger.info(f" ==> Nb total rows of DPAE that have been used : {nb_total_rows_dpae_used} rows")
            logger.info("\n-------------------------")

            i += 1

        return self.path_to_csv, df_dpae_act

def run_main():
    join_act_log = JoinActivityLogsDPAE()
    join_act_log.set_most_recent_dpae_file()
    join_act_log.join_dpae_activity_logs()

if __name__ == '__main__':
    run_main()


import numpy as np
from scipy.stats import gumbel_r
from typing import List

from bioflow.utils.log_behavior import get_logger

log = get_logger(__name__)

def get_neighboring_degrees(degree: int,
                            max_array: np.array,
                            nearest_degrees: int = 0,  # currently inactive
                            min_nodes: int = 10) -> List[float]:
    """
    Recovers the maximum flow achieved by nodes of a given degree for each run. On case the user
    requests it with nearest_degrees or min_nodes parameters, also recovers maximum flow
    values for the nodes of similar degrees or looks fro flow values in nearest degrees until
    at least `min_nodes` are found
    `nearest_degrees`

    :param degree: degree of the nodes
    :param max_array: maximum nodes for a given degree in each run
    :param nearest_degrees: the minimum number of the nearest gedgrees to look for
    :param min_nodes: the minimum number of nodes until which to look for neighbours
    :return:
    """

    max_set = max_array[:, max_array[1, :] == degree]
    max_set_red = max_set[0, :].tolist()

    if len(max_set_red) < min_nodes:
        temp_deg_plus = degree
        temp_deg_minus = degree

        while len(max_set_red) < min_nodes:
            temp_deg_minus -= 1
            temp_deg_plus += 1

            _max_set = max_array[:, max_array[1, :] == temp_deg_minus]
            max_set_red += _max_set[0, :].tolist()

            _max_set = max_array[:, max_array[1, :] == temp_deg_plus]
            max_set_red += _max_set[0, :].tolist()

    log.debug('deg: %d, list: %s' % (degree, max_set_red))

    return max_set_red

def get_p_val_by_gumbel(entry: np.array,
                        max_set_red: List[float]) -> np.array:
    """
    Recovers the statistical significance (p-value equivalent) by performing a gumbel test

    :param entry: the values achieved in the real hits information flow computation
    :param max_set_red: background set of maximum values achieved during blanc sampling runs
    :return:
    """

    params = gumbel_r.fit(max_set_red)
    mu = params[-2]
    beta = params[-1]

    frozen_gumbel = gumbel_r(loc=mu, scale=beta)
    a_95_low, a_95_high = frozen_gumbel.interval(0.95)

    log.debug('gumbel_r fit: mu %.2f, beta: %.2f, 95 alpha: %.2f, .%.2f'
              % (mu, beta, a_95_low, a_95_high))

    p_vals = 1 - frozen_gumbel.cdf(entry)

    return p_vals


from collections import defaultdict
import heapq
from sys import setrecursionlimit
setrecursionlimit(100000)
k = int(input())
g = defaultdict(list)
for i in range(k):
    g[i].append(((i+1)%k, 1))
    g[i].append(((i*10)%k, 0))
dist = [float("Inf")] * k
dist[1] = 0
q = list(zip(dist, range(k)))
heapq.heapify(q)
while len(q) > 0:
    d, v = heapq.heappop(q)
    for u, w in g[v]:
        if dist[u] > d + w:
            dist[u] = d + w
            heapq.heappush(q, (dist[u], u))
print(dist[0]+1)


import json
import re
import time
import hashlib

def run(API, environ, indata, session):

    # We need to be logged in for this!
    if not session.user:
        raise API.exception(403, "You must be logged in to use this API endpoint! %s")

    method = environ['REQUEST_METHOD']
    dOrg = session.user['defaultOrganisation'] or "apache"

    # Are we adding a view?
    if method == 'PUT':
        viewID = hashlib.sha224( ("%s-%s-%s" % (time.time(), session.user['email'], dOrg) ).encode('utf-8') ).hexdigest()
        sources = indata.get('sources', [])
        name = indata.get('name', "unknown view")
        public = indata.get('public', False)
        if public:
            if not (session.user['userlevel'] == 'admin' or dOrg in session.user['ownerships']):
                raise API.exception(403, "Only owners of an organisation may create public views.")
        doc = {
            'id': viewID,
            'email': session.user['email'],
            'organisation': dOrg,
            'sourceList': sources,
            'name': name,
            'created': int(time.time()),
            'publicView': public
        }
        session.DB.ES.index(index=session.DB.dbname, doc_type="view", id = viewID, body = doc)
        yield json.dumps({'okay': True, 'message': "View created"})

    # Are we editing (patching) a view?
    if method == 'PATCH':
        viewID = indata.get('id')
        if viewID and session.DB.ES.exists(index=session.DB.dbname, doc_type="view", id = viewID):
            doc = session.DB.ES.get(index=session.DB.dbname, doc_type="view", id = viewID)
            if session.user['userlevel'] == 'admin' or doc['_source']['email'] == session.user['email']:
                sources = indata.get('sources', [])
                doc['_source']['sourceList'] = sources
                session.DB.ES.index(index=session.DB.dbname, doc_type="view", id = viewID, body = doc['_source'])
                yield json.dumps({'okay': True, 'message': "View updated"})
            else:
                raise API.exception(403, "You don't own this view, and cannot edit it.")
        else:
            raise API.exception(404, "We couldn't find a view with this ID.")

    # Removing a view?
    if method == 'DELETE':
        viewID = indata.get('id')
        if viewID and session.DB.ES.exists(index=session.DB.dbname, doc_type="view", id = viewID):
            doc = session.DB.ES.get(index=session.DB.dbname, doc_type="view", id = viewID)
            if session.user['userlevel'] == 'admin' or doc['_source']['email'] == session.user['email']:
                session.DB.ES.delete(index=session.DB.dbname, doc_type="view", id = viewID)
                yield json.dumps({'okay': True, 'message': "View deleted"})
            else:
                raise API.exception(403, "You don't own this view, and cannot delete it.")
        else:
            raise API.exception(404, "We couldn't find a view with this ID.")

    if method in ['GET', 'POST']:
        # Fetch all views for default org

        res = session.DB.ES.search(
                index=session.DB.dbname,
                doc_type="view",
                size = 5000,
                body = {
                    'query': {
                        'term': {
                            'email': session.user['email']
                        }
                    }
                }
            )

        # Are we looking at someone elses view?
        if indata.get('view'):
            viewID = indata.get('view')
            if session.DB.ES.exists(index=session.DB.dbname, doc_type="view", id = viewID):
                blob = session.DB.ES.get(index=session.DB.dbname, doc_type="view", id = viewID)
                if blob['_source']['email'] != session.user['email'] and not blob['_source']['publicView']:
                    blob['_source']['name'] += " (shared by " +  blob['_source']['email'] + ")"
                    res['hits']['hits'].append(blob)
        sources = []

        # Include public views??
        if not indata.get('sources', False):
            pres = session.DB.ES.search(
                    index=session.DB.dbname,
                    doc_type="view",
                    size = 5000,
                    body = {
                        'query': {
                            'bool': {
                                'must': [
                                    {'term':
                                        {
                                            'publicView': True
                                        }
                                    },
                                    {
                                        'term': {
                                            'organisation': dOrg
                                        }
                                    }
                                ]
                            }
                        }
                    }
                )
            for hit in pres['hits']['hits']:
                if hit['_source']['email'] != session.user['email']:
                    hit['_source']['name'] += " (shared view)"
                    res['hits']['hits'].append(hit)

        for hit in res['hits']['hits']:
            doc = hit['_source']
            if doc['organisation'] != dOrg:
                continue
            if indata.get('quick'):
                xdoc = {
                    'id': doc['id'],
                    'name': doc['name'],
                    'organisation': doc['organisation']
                    }
                sources.append(xdoc)
            else:
                sources.append(doc)

        allsources = []
        if indata.get('sources', False):
            res = session.DB.ES.search(
                index=session.DB.dbname,
                doc_type="source",
                size = 5000,
                body = {
                    'query': {
                        'term': {
                            'organisation': dOrg
                        }
                    }
                }
            )
            for zdoc in res['hits']['hits']:
                doc = zdoc['_source']
                xdoc = {
                    'sourceID': doc['sourceID'],
                    'type': doc['type'],
                    'sourceURL': doc['sourceURL']
                    }
                allsources.append(xdoc)

        JSON_OUT = {
            'views': sources,
            'sources': allsources,
            'okay': True,
            'organisation': dOrg
        }
        yield json.dumps(JSON_OUT)


#!/usr/bin/env python
"""
Valid Perfect Square

Given a positive integer num, write a function which returns True if num is a
perfect square else False.

Note: Do not use any built-in library function such as sqrt.

Example 1:

Input: 16
Output: true

Example 2:

Input: 14
Output: false
"""

class Solution:
    def isPerfectSquare(self, num: int) -> bool:
        if num == 1:
            return True
        low = 1
        high = num // 2
        while low <= high:
            mid = (low + high) // 2
            guess = mid * mid
            if guess < num:
                low = mid + 1
            elif guess == num:
                return True
            else:  # guess > num
                high = mid - 1
        return False


#Global Variables

Row1 = ['-','-','-']
Row2 = ['-','-','-']
Row3 = ['-','-','-']
Board = [Row3,Row2,Row1]
Coloum = []

Xwin = ['X','X','X']
Ywin = ['X','X','X']

win = False
turnnum = 0

#Prints current board state
def printboard():
  for rows in range(len(Board)):
        print(Board[rows])

#Promted player input for human turn
def playerturn():
  global turnnum
  plrinput = input('\"X\" input position (1-9): ')
  intinput = int(plrinput)
  #Sets input to board space
  if intinput>9:
    print('Please choose a valid number.')
  elif intinput<1:
    print('Please choose a valid number.')
  elif intinput < 7:
    turnnum = turnnum+1
    if intinput >3:
       Row2[intinput-4] = 'X'
    else:
      Row1[intinput-1] = 'X'
  else:
    turnnum=turnnum+1
    Row3[intinput - 7] = 'X'

#Checks game for win state
def winstate():
  coloum = 0
  global win
  for row in Board:
    if row == Xwin:
      win = True
      print('X wins!')
    elif row == Ywin:
      win = True
      print('Y wins!')
  while coloum<3:
    Coloum.append(Row1[coloum])
    Coloum.append(Row2[coloum])
    Coloum.append(Row3[coloum])
    if Coloum == Xwin:
      win = True
      print('X wins!')
    elif Coloum == Ywin:
      win = True
      print('Y wins!')
    Coloum.clear()
    coloum = coloum + 1
#Plays game while not in win state
def boardplaystate():
  global turnnum
  global win
  while win == False:
    playerturn()
    printboard()
    winstate()

printboard()
boardplaystate()


from bayserver_core.util.reusable import Reusable
from bayserver_core.bay_log import BayLog
from bayserver_core.sink import Sink
from bayserver_core.util.object_factory import ObjectFactory
from bayserver_core.util.string_util import StringUtil

class ObjectStore(Reusable):

    def __init__(self, factory=None):
        self.free_list = []
        self.active_list = []
        self.factory = factory

    ######################################################
    # implements Reusable
    ######################################################

    def reset(self):
        if len(self.active_list) > 0:
            BayLog.error("BUG?: There are %d active objects: %s",  len(self.active_list), self.active_list)

            # for security
            self.free_list.clear()
            self.active_list.clear()

    ######################################################
    # Other methods
    ######################################################

    def rent(self):
        if len(self.free_list) == 0:
            if isinstance(self.factory, ObjectFactory):
                obj = self.factory.create_object()
            else:
                # lambda
                obj = self.factory()
        else:
            obj = self.free_list.pop()

        if obj is None:
            raise Sink()

        self.active_list.append(obj)

        return obj

    def Return(self, obj, reuse=True):
        if obj in self.free_list:
            raise Sink("This object already returned: %s", obj)

        if obj not in self.active_list:
            raise Sink("This object is not active: %s", obj)

        self.active_list.remove(obj)

        if reuse:
            self.free_list.append(obj)

        obj.reset()

    def print_usage(self, indent):
        BayLog.info("%sfree list: %d", StringUtil.indent(indent), len(self.free_list));
        BayLog.info("%sactive list: %d", StringUtil.indent(indent), len(self.active_list));
        if BayLog.debug_mode():
            for obj in self.active_list:
                BayLog.debug("%s%s", StringUtil.indent(indent + 1), obj)


'''
Created on May 29, 2015

@author: Adrian
'''
import pygame
from pygame.locals import *

#Derive your class from the Sprite super class
class PowerUps(pygame.sprite.Sprite):
    def __init__(self, x, y, vx, vy, h_walls_list, v_walls_list, url, speedUp, slowDown):
        """
            Creates a power up object to increase or decrease ball speed
            :param x: x location of power up origin
            :param y: y location of power up origin
            :param vx: velocity of the power up in the x direction
            :param vy: velocity of the power up in the y direction
            :param h_walls_list: list of pygroup horizontal walls for collisions
            :param v_walls_list: list of pygroup vertical walls for collisions
            :param url: url for power up pictures
            :param speedUp: boolean that signals speeding up of the power ups
            :param slowDown: boolean that signals slowing down of the power ups

        """
        # Don't forget to call the super constructor
        super().__init__();
        self.image = pygame.image.load(url).convert()

        # Set the color that should be transparent
#         self.image.set_colorkey(pygame.Color(0, 0, 0))
        self.image.set_colorkey(pygame.Color(255, 255, 255))

        # Required for collision detection
        # HINT: You will need this for the lab
        self.rect = self.image.get_rect()

        self.rect.x = x
        self.rect.y = y
        self.vx = vx
        self.vy = vy
        self.speedUp = speedUp
        self.slowDown = slowDown

        self.h_walls = h_walls_list
        self.v_walls = v_walls_list

    def update(self, balls, powerUps):
        """
            Responsible testing whether the power ups collide with horizontal
            walls, vertical walls, or ball.
            :param balls: list of balls to test for powerUp to ball collision
            :param powerUps: list of powerUps to test for powerUp to powerUps collision
        """
        # Move up/down
        self.rect.y += self.vy
        # Move left/right
        self.rect.x += self.vx

        # Did this update cause us to hit a wall?
        tb_hit_list = pygame.sprite.spritecollide(self, self.h_walls, False)
        for p in tb_hit_list:
            self.vy *= -1

        lr_hit_list = pygame.sprite.spritecollide(self, self.v_walls, False)
        for p in lr_hit_list:
            self.vx *= -1

        collisionList = pygame.sprite.spritecollide(self, powerUps, False)
        for c in collisionList: #if two edges are barely overlapping, then the two objects are colliding on those edges
            if (0 < self.rect.y - c.rect.y + c.image.get_height() <= 10) or (0 < self.rect.y + self.image.get_height() - c.rect.y <= 10):
                self.vy *= -1 #if collide, undo the move in the collion's direction
                self.rect.y += 2*self.vy
                c.vy *= -1 #change the direction of the other sprite, prevents collision when c moves
            if (0 < self.rect.x - c.rect.x + c.image.get_width() <= 10) or (0 < self.rect.x + self.image.get_width() - c.rect.x <= 10):
                self.vx *= -1
                self.rect.x += 2*self.vx
                c.vx *= -1


# Fibonacci Heap Implementation
from node import Node

class FibonacciHeap:
    # Constructor
    def __init__(self):
        self.min = None
        self.n = 0

    # Insert method
    def insert(self, node):
        node.left = node
        node.right = node
        node.degree = 0
        node.mark = False
        self.link(node)
        if self.min is None:
            self.min = node
        elif node.key < self.min.key:
            self.min = node

    # Link method
    def link(self, y):
        x = self.min
        if x is None:
            self.min = y
        elif x.key > y.key:
            self.min = y
            x, y = y, x
        y.left = y.left.right = y
        y.parent = x
        y.mark = False
        x.degree += 1
        y.child = x.child if x.child is not None else y
        x.child = y

    # Cascading Cut method
    def cascading_cut(self, y):
        z = y.parent
        if z is not None:
            if y.mark is False:
                y.mark = True
            else:
                self.cut(y, z)
                self.cascading_cut(z)

    # Cut method
    def cut(self, x, y):
        x.left.right = x.right
        x.right.left = x.left
        y.degree -= 1
        if y.child is x:
            y.child = x.right if x.right is not x else None
        if y.degree == 0:
            y.child = None
        x.left = x
        x.right = x
        x.parent = None
        x.mark = False
        self.link(y)

    # Decrease Key method
    def decrease_key(self, x, k):
        if k > x.key:
            return
        x.key = k
        y = x.parent
        if y is not None and x.key < y.key:
            self.cut(x, y)
            self.cascading_cut(y)
        if x.key < self.min.key:
            self.min = x

    # Extract Minimum method
    def extract_min(self):
        z = self.min
        if z is not None:
            if z.child is not None:
                x = z.child
                y = x.right
                x.left = x.right = x
                self.min = x
                self.consolidate()
                self.min = self.meld(self.min, y)
            self.min = z.right if z.right is not z else None
            z.left.right = z.right
            z.right.left = z.left
        return z

    # Consolidate method
    def consolidate(self):
        A = [None] * self.n
        w = self.min
        while w is not None:
            x = w
            d = x.degree
            while A[d] is not None:
                y = A[d]
                if x.key > y.key:
                    x, y = y, x
                self.link(y)
                A[d] = None
                d += 1
            A[d] = x
            w = w.right
            w = self.min if w is x else w
            d += 1

    # Meld method
    def meld(self, h1, h2):
        if h1 is None:
            return h2
        if h2 is None:
            return h1
        if h1.key > h2.key:
            h1, h2 = h2, h1
        h1.right.left = h1.left
        h1.left.right = h1.right
        h1.left = h1.right = h1
        self.link(h2)
        if h1.key < self.min.key:
            self.min = h1
        return h1

    # Delete method
    def delete(self, x):
        self.decrease_key(x, float("-inf"))
        self.extract_min()

    # Empty method
    def empty(self):
        return self.min is None

    # Size method
    def size(self):
        return self.n

    # Print method
    def print(self):
        w = self.min
        while w is not None:
            print(w.key)
            w = w.right
            w = self.min if w is x else w

# Test
if __name__ == "__main__":
    fh = FibonacciHeap()
    # Create nodes
    n1 = Node(1)
    n2 = Node(2)
    n3 = Node(3)
    n4 = Node(4)
    n5 = Node(5)

    # Insert nodes
    fh.insert(n1)
    fh.insert(n2)
    fh.insert(n3)
    fh.insert(n4)
    fh.insert(n5)


"""
Test 3d plotting and creating test matrices
"""
import os
import numpy as np

import matplotlib.pyplot as plt
import matplotlib
matplotlib.use('agg')

from nextorch import plotting, utils

dir_path = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(dir_path, 'test_3d.png')

# test adding a 2d plane to a 3d space
fig = plt.figure(figsize=(6, 6))
ax = fig.add_subplot(111, projection='3d')
plotting.add_x_slice_2d(ax, 10, [1,100], [2,4])
fig.savefig(save_path, bbox_inches="tight")

# switch back to interactive mode
# matplotlib.use('TkAgg')

# test create full X functions
X_ranges = [[0,1], [0,2], [0,3], [2,3]]
X_test_2d, _, _ = utils.create_full_X_test_2d(X_ranges=X_ranges, x_indices=[0,1])
X_test_1d = utils.create_full_X_test_1d(X_ranges=X_ranges, x_index=0, baseline = 'center')

def test_X_2d_dimensions():
    # Test on two-dimensional testing matrix
    assert X_test_2d.shape[0] == 41**2
    assert X_test_2d.shape[1] == len(X_ranges)

def test_X_1d_values():
    # Test on one-dimensional testing matrix
    x_real_0 = np.linspace(0,1, 41)
    assert np.all(x_real_0 == X_test_1d[:,0])
    assert np.all(X_test_1d[:,1:] == 0.5)


import csv
import json

f = open('D:\\REST API\\swagger\\validator-badge\\result\\security-v2.03781946110160745780.csv','r',newline='')
reader = csv.reader(f)
f1 = open('D:\\REST API\\swagger\\validator-badge\\result\\security-v3.03728676635139631361.csv','r',newline='')
reader1 = csv.reader(f1)
secudict={}
nullSum=0
sum=0
for row in reader:
    sum+=1
    if row[1]=="":
        nullSum += 1
        continue
    for cate in row[1:]:
        if cate != "":
            if cate in secudict :
                secudict[cate]+=1
            else:
                secudict[cate]=1
for row in reader1:
    sum += 1
    if row[1]=="":
        nullSum += 1
        continue
    for cate in row[1:]:
        if cate != "":
            if cate in secudict :
                secudict[cate]+=1
            else:
                secudict[cate]=1

print(secudict)
print(nullSum)
print(sum)

f.close()
f1.close()

# jsObj = json.dumps(secudict)
# f3 = open('D:\\REST API\\swagger\\validator-badge\\result\\securityStatistics.json', 'w')
# f3.write(jsObj)
# f3.close()


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from mpl_toolkits.axes_grid1 import make_axes_locatable

def error_bars(x: str, y: str, data: pd.DataFrame, central_tendency: str = 'median',
               error_method: str = 'percentile',
               sections: dict = None, colors: list = None,
               xlim: tuple = None, cax_percentage: str = "40%"):
    """
    Plot fancy error bar
    Design is basically copied from this recent Covid-19 paper: https://www.medrxiv.org/content/10.1101/2021.03.25.21254330v1.full.pdf

    :param x: name of dataframe column containing values to be plotted
    :param y: name of dataframe column containing bar groups
    :param data: pandas dataframe
    :param central_tendency: 'median' or 'mean'
    :param error_method: 'percentile' or 'standard_deviation'
    :param sections: dictionary of section names and corresponding group names
    :param colors: list of colors, length should correspond to number of sections or groups
    :param xlim: x axis limits
    :param cax_percentage: how big do you want the y labels to be
    :return:
        fig: matplotlib figure
        ax: matplotlib main plot axis
        cax: additional matplotlib axis for y tick label
    """
    n_sections = None
    y_coord_sections = None

    # get colors and y coords
    if sections:
        n_sections = len(sections.keys())

        if colors is None:
            colors = sns.color_palette('colorblind', n_sections)
        else:
            if len(colors) != n_sections:
                raise RuntimeError("Number of colors has to match number of sections.")

        colors_bars = list()
        y_coord_bars = list()
        y_coord_sections = list()
        y_coord = 0
        names = list()
        for section_i, section_bars in enumerate(sections.values()):
            y_coord_sections.append(y_coord)
            y_coord -= 0.25

            for bar_name in section_bars:
                names.append(bar_name)
                y_coord -= 0.5
                y_coord_bars.append(y_coord)
                y_coord -= 0.5
                colors_bars.append(colors[section_i])
            y_coord -= 0.25
        n_bars = len(names)
    else:
        names = data[y].unique()
        n_bars = len(names)
        y_coord = 0
        y_coord_bars = list()
        for bar_name in names:
            y_coord -= 0.5
            y_coord_bars.append(y_coord)
            y_coord -= 0.5

        if colors is None:
            colors_bars = sns.color_palette('colorblind', n_bars)
        else:
            if len(colors) != n_bars:
                raise RuntimeError("Number of colors has to match number of groups in data.")
            colors_bars = colors

    # compute central tendency
    if central_tendency == 'median':
        ct_method = np.median
    elif central_tendency == 'mean':
        ct_method = np.mean
    else:
        raise NotImplementedError("central_tendency has to be either 'mean' or 'median'.")
    m = np.zeros(n_bars)
    for i, b in enumerate(names):
        m[i] = ct_method(data.loc[data[y] == b, x].values)

    # compute error bars
    if error_method == 'percentile':
        li, lq, ui, uq = np.zeros(n_bars), np.zeros(n_bars), np.zeros(n_bars), np.zeros(n_bars)
        for i, b in enumerate(names):
            li[i], lq[i], uq[i], ui[i] = np.percentile(data.loc[data[y] == b, x].values,
                                                       [2.5, 25, 75, 97.5])
    elif error_method == 'standard_deviation':
        sd = np.zeros(n_bars)
        for i, b in enumerate(names):
            sd[i] = np.std(data.loc[data[y] == b, x].values)
        li, lq = None, m - sd
        ui, uq = None, m + sd
    elif error_method == 'confidence_interval':
        raise NotImplementedError("error_method 'confidence_interval' is not yet supported.")
    elif error_method == 'None':
        li, lq = None, m
        ui, uq = None, m
    else:
        raise NotImplementedError("error_method {} is not supported.".format(error_method))

    # start drawing the figure
    # todo: check if we really need GridSpec here
    fig = plt.figure(constrained_layout=True, figsize=(5, 4), dpi=400)
    gs = gridspec.GridSpec(ncols=3, nrows=4, figure=fig)
    ax = fig.add_subplot(gs[:, :])

    # plot error bars
    for bar_i in range(n_bars):
        y_coord = y_coord_bars[bar_i]
        if li:
            ax.plot([li[bar_i], ui[bar_i]], [y_coord, y_coord], color=colors_bars[bar_i], alpha=0.35, linewidth=1,
                    solid_capstyle='round')
        ax.plot([lq[bar_i], uq[bar_i]], [y_coord, y_coord], color=colors_bars[bar_i], alpha=0.85, linewidth=1,
                solid_capstyle='round')

    if xlim is None:
        xmin, xmax = ax.get_xlim()
    else:
        xmin, xmax = xlim

    # plot grey shadow for bars
    for bar_i in range(n_bars):
        start = y_coord_bars[bar_i] + 0.48
        end = y_coord_bars[bar_i] - 0.48
        ax.fill_between(
            [xmin, xmax],
            [end, end],
            [start, start],
            color='#939596', alpha=0.15, linewidth=0)

    # plot grey shadow for section dividers
    if sections:
        for section_i in range(n_sections):
            start = y_coord_sections[section_i] + 0.23
            end = y_coord_sections[section_i] - 0.23
            ax.fill_between(
                [xmin, xmax],
                [end, end],
                [start, start],
                color='#939596', alpha=0.25, linewidth=0)

    # plot scatter points for central tendency
    ax.scatter(m, y_coord_bars, marker="o", color=colors_bars, s=6, edgecolors=colors_bars, facecolor="white",
               zorder=5)

    # style
    y_min = y_coord_bars[-1] - 0.5
    ax.set_ylim([y_min, 0.25])
    ax.set_xlim([xmin, xmax])
    ax.spines['bottom'].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.spines["left"].set_visible(False)
    ax.spines["top"].set_visible(False)
    ax.yaxis.set_ticks_position('none')
    ax.xaxis.set_ticks_position('none')

    # plot additional axis to get fancy y tick labels
    divider = make_axes_locatable(ax)
    # 50% might be adjusted depending on the length of the y tick labels
    cax = divider.append_axes("left", size=cax_percentage, pad=0)

    cax.get_xaxis().set_visible(False)
    cax.get_yaxis().set_visible(False)
    cax.spines['bottom'].set_visible(False)
    cax.spines["right"].set_visible(False)
    cax.spines["left"].set_visible(False)
    cax.spines["top"].set_visible(False)
    cax.set_ylim([y_min, 0.25])
    cax.set_xlim([xmin, xmax])

    # plot grey shadow for bars
    for bar_i in range(n_bars):
        start = y_coord_bars[bar_i] + 0.48
        end = y_coord_bars[bar_i] - 0.48
        cax.fill_between(
            [xmin, xmax],
            [end, end],
            [start, start],
            color=colors_bars[bar_i], alpha=0.15, linewidth=0)
        cax.text(xmax - 0.05 * (xmax - xmin), y_coord_bars[bar_i], names[bar_i],
                 verticalalignment='center', horizontalalignment='right',
                 fontsize=7)

    # plot grey shadow for section dividers
    if sections:
        section_names = list(sections.keys())
        for section_i in range(n_sections):
            start = y_coord_sections[section_i] + 0.23
            end = y_coord_sections[section_i] - 0.23
            cax.fill_between(
                [xmin, xmax],
                [end, end],
                [start, start],
                color=colors[section_i], alpha=0.25, linewidth=0)
            cax.text(xmax - 0.05 * (xmax - xmin), y_coord_sections[section_i], section_names[section_i],
                     verticalalignment='center', horizontalalignment='right',
                     fontsize=5)

    return fig, ax, cax

if __name__ == "__main__":
    tips = sns.load_dataset("tips")

    sections = {'Working Days': ['Thur', 'Fri'],
                'Weekend': ['Sat', 'Sun']}

    error_bars(x='total_bill', y='day', sections=sections,
               data=tips, error_method='percentile')
    plt.savefig('total_bills_per_day_sections.png')
    plt.show()

    error_bars(x='total_bill', y='day',
               data=tips, error_method='percentile')
    plt.savefig('total_bills_per_day.png')
    plt.show()

    error_bars(x='tip', y='time', data=tips, error_method='percentile')
    plt.savefig('total_tips_per_time.pdf')
    plt.show()


import sys
sys.setrecursionlimit(10**6)
input = sys.stdin.readline

N = int(input())

tree = [[] for _ in range(N+1)]
visited = [False] * (N+1)
answer = [0] * (N+1)   #부모노드 저장

for _ in range(N-1):
    n1, n2 = map(int, input().split())
    tree[n1].append(n2)
    tree[n2].append(n1)

def DFS(start):
    visited[start] = True
    for node in tree[start]:
        if not visited[node]:
            answer[node] = start
            DFS(node)

DFS(1)

for i in range(2, N+1):
    print(answer[i])

백준/Silver/11725. 트리의 부모 찾기/트리의 부모 찾기.py

트리의 부모 찾기.py


"""
给定一个字符串 s ，找出 至多 包含 k 个不同字符的最长子串 T。

示例 1:

输入: s = "eceba", k = 2
输出: 3
解释: 则 T 为 "ece"，所以长度为 3。
示例 2:

输入: s = "aa", k = 1
输出: 2
解释: 则 T 为 "aa"，所以长度为 2。

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/longest-substring-with-at-most-k-distinct-characters
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
"""
import collections

# 2021.04.19 重温双指针
class Solution:
    def lengthOfLongestSubstringKDistinct(self, s: str, k: int) -> int:
        dic = collections.defaultdict(int)
        left = right = res = 0
        while right < len(s):
            dic[s[right]] += 1
            while len(dic) > k:
                dic[s[left]] -= 1
                if dic[s[left]] == 0:
                    del dic[s[left]]
                left += 1
            res = max(res, right-left+1)
            right += 1
        return res

340_至多包含K个不同字符的最长子串.py


# 세 변의 길이가 모두 자연수 {a, b, c}인 직각삼각형의 둘레를 p 로 둘 때, p = 120 을 만족하는 직각삼각형은 아래와 같이 세 개가 있습니다.
# {20, 48, 52}, {24, 45, 51}, {30, 40, 50}
# 1000 이하의 둘레 p에 대해서, 직각삼각형이 가장 많이 만들어지는 p의 값은 얼마입니까?

# 노창균t

def myn(n):
    my_lenth =0
    my_list =[]
    for x in range(1,(n//3) +1):
        for y in range((n-3*x)//2 +1):
            z = n-3*x -2*y
            a,b,c= x, x+y, x+y+z
            if a**2 +b**2 ==c **2:
                my_list.append((a,b,c))
                my_lenth += 1
    return my_lenth, my_list

max_length , max_list, max_i =0,[],0
for i in range(1,1001):
    result = myn(i)
    if result[0] >max_length:
        max_length = result[0]
        max_list = result[1]
        max_i =i

print("가장 큰 수는 {}이고 순서쌍의 수는 {} 이고, 이 리스트는 {}이다".format(max_i,max_length, max_list))

# 가장 큰 수는 1000이고 순서쌍의 수는 8 이고,
# 이 리스트는 [(40, 399, 401), (56, 390, 394), (105, 360, 375), (120, 350, 370), (140, 336, 364), (168, 315, 357), (210, 280, 350), (240, 252, 348)]이다


'''
Programa que pregunta: nombre, apellido, edad, dirección, fecha de nacimiento, teléfono y correo electrónico.
Además, debe sumar la edad + año de nacimiento
'''
name = input('¿Cuál es su nombre de pila?')
surname = input('¿y su apellido?')
age = int(input('¿Cuántos años tiene?'))
address = input('¿Cuál es su dirección?')
print('Introduzca su fecha de nacimiento con el formato siguiente: día/mes/año')
date_entry = input()
day, month, year = map(int, date_entry.split('/'))
phone = input('¿Cuál es su teléfono?')
email = input('¿Cómo es su correo electrónico?')
yearNow = str(age+year)
print('Si ya ha sido mi cumple, estamos en el año ' + yearNow + 'Si no, un año más. Hola, me llamo ' + name +' '+ surname + '. vivo en '
 + address + '. Nací el ' + date_entry + '. Llámenme al número ' + phone + ' o escríbame a la dirección ' + email + '.')


input_elevation_map_alpha_string = """
Sabqponm
abcryxxl
accszExk
acctuvwj
abdefghi"""

ROW = 0
COL = 1

given_elevation_map_alpha = [x.strip() for x in input_elevation_map_alpha_string.split("\n") if not x.strip() == ""]

size = (len(given_elevation_map_alpha), len(given_elevation_map_alpha[0]))

given_start: (int, int) = (None, None)
given_end: (int, int) = (None, None)

def replace_char_in_string(string: str, pos: int, char: str) -> str:
    return string[:pos] + char + string[pos + 1:]

assert replace_char_in_string("flip", 2, "o") == "flop"

for row in range(size[ROW]):
    for col in range(size[COL]):
        if given_elevation_map_alpha[row][col] == "S":
            given_start = (row, col)
            given_elevation_map_alpha[row] = replace_char_in_string(given_elevation_map_alpha[row], col, "a")
        if given_elevation_map_alpha[row][col] == "E":
            given_end = (row, col)
            given_elevation_map_alpha[row] = replace_char_in_string(given_elevation_map_alpha[row], col, "z")

given_elevation_map = [[ord(y) - ord("a") for y in x.strip()] for x in given_elevation_map_alpha]

def find_shortest_path(elevation_map, start: (int, int), end: (int, int), climbing: bool) -> int:
    steps_to_reach: list[list[int | None]] = [[None for _ in range(size[COL])] for _ in range(size[ROW])]
    steps_to_reach[start[ROW]][start[COL]] = 0

    reached = [[False for _ in range(size[COL])] for _ in range(size[ROW])]
    reached[start[ROW]][start[COL]] = True

    def debug_print():
        for row in range(size[ROW]):
            output = ""
            for col in range(size[COL]):
                if row == start[ROW] and col == start[COL]:
                    output += "S"
                else:
                    if end and row == end[ROW] and col == end[COL]:
                        output += "E"
                    else:
                        if steps_to_reach[row][col]:
                            output += chr(ord("a") + steps_to_reach[row][col] - 1)
                        else:
                            output += "."
            print(output)

    stalled = False
    current_distance = 1
    if climbing:
        step_condition = lambda unreached_elevation, reached_elevation: unreached_elevation <= reached_elevation + 1
    else:
        step_condition = lambda unreached_elevation, reached_elevation: reached_elevation <= unreached_elevation + 1

    while not (end and reached[end[ROW]][end[COL]]) and not stalled:
        stalled = True
        for col in range(size[COL]):
            in_first_col = (col == 0)
            in_last_col = (col == size[COL] -1)
            for row in range(size[ROW]):
                in_first_row = (row == 0)
                in_last_row = (row == size[ROW] - 1)
                if row == 2 and col == 4:
                    pass
                if not reached[row][col]:
                    this_cell_elevation = elevation_map[row][col]
                    this_cell_now_reachable = False
                    if not in_first_col and steps_to_reach[row][col - 1] == current_distance - 1:
                        if step_condition(this_cell_elevation, elevation_map[row][col - 1]):
                            this_cell_now_reachable = True
                    if not in_last_col and steps_to_reach[row][col + 1] == current_distance - 1:
                        if step_condition(this_cell_elevation, elevation_map[row][col + 1]):
                            this_cell_now_reachable = True
                    if not in_first_row and steps_to_reach[row - 1][col] == current_distance - 1:
                        if step_condition(this_cell_elevation, elevation_map[row - 1][col]):
                            this_cell_now_reachable = True
                    if not in_last_row and steps_to_reach[row + 1][col] == current_distance - 1:
                        if step_condition(this_cell_elevation, elevation_map[row + 1][col]):
                            this_cell_now_reachable = True
                    if this_cell_now_reachable:
                        steps_to_reach[row][col] = current_distance
                        reached[row][col] = True
                        stalled = False
                        if not climbing and elevation_map[row][col] == 0:
                            end = (row, col)
        current_distance += 1
    if end:
       return steps_to_reach[end[ROW]][end[COL]]

print("Part 1: Fewest steps to reach = %d" % find_shortest_path(given_elevation_map, given_start, given_end, True))
print("Part 2: Fewest steps to reach = %d" % find_shortest_path(given_elevation_map, given_end, None, False))


import logging
import os
import sys
from operator import itemgetter as ig

import pytz
import re
from datetime import datetime
from functools import lru_cache, partial
from future.utils import lmap, lfilter

from foxylib.tools.collections.collections_tool import lchain, l_singleton2obj
from foxylib.tools.collections.groupby_tool import gb_tree_global
from foxylib.tools.collections.iter_tool import iter2singleton
from foxylib.tools.function.function_tool import FunctionTool
from foxylib.tools.function.warmer import Warmer
from foxylib.tools.locale.locale_tool import LocaleTool
from foxylib.tools.regex.regex_tool import RegexTool, MatchTool
from foxylib.tools.string.string_tool import str2strip, format_str
from henrique.main.document.culture.culture_entity import CultureEntity
from foxylib.tools.entity.entity_tool import FoxylibEntity
from henrique.main.document.henrique_entity import HenriqueEntity
from henrique.main.document.port.port import Port
from henrique.main.document.port.port_entity import PortEntity
from henrique.main.document.price.mongodb.marketprice_doc import MarketpriceDict, MarketpriceDoc
from henrique.main.document.price.rate.rate_entity import RateEntity
from henrique.main.document.price.trend.trend_entity import Trend, TrendEntity
from henrique.main.document.server.server import Server
from henrique.main.document.tradegood.tradegood_entity import TradegoodEntity
from henrique.main.singleton.datetime.henrique_datetime import HenriqueDatetime
from henrique.main.singleton.khala.henrique_khala import Rowsblock, HenriquePacket
from henrique.main.singleton.logger.henrique_logger import HenriqueLogger
from khala.document.channel_user.channel_user import ChannelUser
from khala.document.chatroom.chatroom import Chatroom
from khala.document.packet.packet import KhalaPacket

FILE_PATH = os.path.realpath(__file__)
FILE_DIR = os.path.dirname(FILE_PATH)

MODULE = sys.modules[__name__]
WARMER = Warmer(MODULE)

class Portlike:
    @classmethod
    def entity_types(cls):
        return {PortEntity.entity_type(), CultureEntity.entity_type()}

    @classmethod
    def entity_type2is_portlike(cls, entity_type):
        return entity_type in cls.entity_types()

    @classmethod
    def entity_portlike2port_codenames(cls, entity_portlike):
        entity_type = FoxylibEntity.entity2type(entity_portlike)
        if entity_type == PortEntity.entity_type():
            return [FoxylibEntity.entity2value(entity_portlike)]

        if entity_type == CultureEntity.entity_type():
            culture_codename = FoxylibEntity.entity2value(entity_portlike)
            port_list = Port.culture2ports(culture_codename)
            return lmap(Port.port2codename, port_list)

        raise RuntimeError({"entity_type": entity_type})

class PriceSkillParameter:
    class Type:
        PORTLIKE = "portlike"
        TRADEGOOD = "tradegood"
        RATE = "rate"
        TREND = "trend"

        @classmethod
        def list(cls): return [cls.PORTLIKE, cls.TRADEGOOD, cls.RATE, cls.TREND, ]

        @classmethod
        def entity_type2parameter_type(cls, entity_type):
            h = {PortEntity.entity_type(): cls.PORTLIKE,
                 CultureEntity.entity_type(): cls.PORTLIKE,
                 TradegoodEntity.entity_type(): cls.TRADEGOOD,
                 RateEntity.entity_type(): cls.RATE,
                 TrendEntity.entity_type(): cls.TREND,
                 }

            return h.get(entity_type)

        @classmethod
        def entity_group2parameter_type(cls, entity_list):
            entity_types = map(FoxylibEntity.entity2type, entity_list)
            return iter2singleton(map(cls.entity_type2parameter_type, entity_types))

    @classmethod
    @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2))
    def pattern_delim(cls):
        return re.compile(r"[,\s]*")

class PriceSkill:
    CODENAME = "price"

    @classmethod
    def lang2description(cls, lang):
        from henrique.main.skill.price.price_skill_description import PriceSkillDescription
        return PriceSkillDescription.lang2text(lang)

    @classmethod
    def dict_lang2text_idk(cls):
        return {"ko": "아몰랑",
                "en": "idk",
                }

    @classmethod
    def lang2text_idk(cls, lang):
        raise NotImplementedError()
        # doc = {MarketpriceDoc.Field.}
        # return cls.rate_trend_date_lang2text(100, Trend.Value.AVERAGE)

    @classmethod
    def lang2text_idk_OLD(cls, lang):
        text_idk = cls.dict_lang2text_idk().get(lang)

        if text_idk is None:
            raise NotImplementedError({"lang": lang})

        return text_idk

    # @classmethod
    # def _rate_trend_created_at2text(cls, rate, trend, created_at):
    #     arrow = Trend.trend2arrow(trend)
    #
    #
    #     return format_str("{} {} ({})", str(rate), arrow)

    @classmethod
    def price_lang2text(cls, price, lang):
        logger = HenriqueLogger.func_level2logger(cls.price_lang2text, logging.DEBUG)

        if price is None:
            return cls.lang2text_idk(lang)

        rate = MarketpriceDoc.price2rate(price)
        trend = MarketpriceDoc.price2trend(price)
        # raise Exception({"price":price})

        channel_user_codename = MarketpriceDoc.price2channel_user(price)
        channel_user = l_singleton2obj(ChannelUser.codenames2channel_users([channel_user_codename]))
        logger.debug({"price":price, "channel_user":channel_user})

        # raise Exception({"price":price})
        created_at = MarketpriceDoc.price2created_at(price) or MarketpriceDoc.created_at_backoff()
        td = datetime.now(tz=pytz.utc) - created_at
        str_timedelta = HenriqueDatetime.timedelta_lang2str(td, lang)

        arrow = Trend.trend2arrow(trend)
        text_out_base = format_str("{}{} @ {}", str(rate), arrow, str_timedelta)

        if td >= HenriqueDatetime.Constant.TIMEDELTA_OUTDATED:
            return text_out_base

        user_alias = ChannelUser.channel_user2alias(channel_user)
        str_user_alias = "[by {}]".format(user_alias)
        text_out = " ".join([text_out_base, str_user_alias])
        return text_out

    @classmethod
    def packet2rowsblocks(cls, packet):
        from henrique.main.skill.price.price_skill_clique import PriceSkillClique as Clique
        Param = PriceSkillParameter

        text = KhalaPacket.packet2text(packet)
        server_codename = HenriquePacket.packet2server(packet)

        chatroom = Chatroom.codename2chatroom(KhalaPacket.packet2chatroom(packet))

        config = {HenriqueEntity.Config.Field.LOCALE: Chatroom.chatroom2locale(chatroom)}
        extractors = Clique.config2extractors(config)
        entity_list = HenriqueEntity.text_extractors2entity_list(text, extractors)

        clique_list = Clique.text_entity_list2clique_list(text, entity_list)
        clique_list_update = lfilter(lambda x: Clique.clique2type(x) == Clique.Type.UPDATE, clique_list)

        # raise Exception({"entity_list": entity_list,
        #                  "clique_list": clique_list,
        #                  "clique_list_update":clique_list_update,
        #                  })
        if clique_list_update:
            Clique.clique_list2update_mongodb(packet, clique_list_update)

        h_port2indexes = Clique.cliques2dict_port2indexes(clique_list)
        h_tradegood2indexes = Clique.cliques2dict_tradegood2indexes(clique_list)

        def _groupby_paramter_type():
            entity_list_portlike = lfilter(lambda x: FoxylibEntity.entity2type(x) in Portlike.entity_types(), entity_list)
            entity_list_tradegood = lfilter(lambda x: FoxylibEntity.entity2type(x) == TradegoodEntity.entity_type(), entity_list)

            if not entity_list_portlike:
                return Param.Type.TRADEGOOD

            if not entity_list_tradegood:
                return Param.Type.PORTLIKE

            if len(h_port2indexes) > 1:
                return Param.Type.TRADEGOOD

            if len(h_tradegood2indexes) > 1:
                return Param.Type.PORTLIKE

            span_portlike = FoxylibEntity.entity2span(entity_list_portlike[0])
            span_tradegood = FoxylibEntity.entity2span(entity_list_tradegood[0])

            if span_portlike[0] < span_tradegood[0]:
                return Param.Type.PORTLIKE
            else:
                return Param.Type.TRADEGOOD

        groupby_parameter_type = _groupby_paramter_type()

        port_tradegood_list = lchain(*map(Clique.clique2port_tradegood_iter, clique_list))
        price_dict = MarketpriceDict.port_tradegood_iter2price_dict(server_codename, port_tradegood_list)

        # raise Exception({"port_tradegood_list": port_tradegood_list,
        #                  "price_dict": price_dict,
        #                  "groupby_parameter_type": groupby_parameter_type,
        #                  })
        chatroom = Chatroom.codename2chatroom(KhalaPacket.packet2chatroom(packet))
        lang = LocaleTool.locale2lang(Chatroom.chatroom2locale(chatroom))
        block_list = cls.port_tradegood_lists2blocks(port_tradegood_list, price_dict, lang, groupby_parameter_type)
        return block_list

    @classmethod
    def port_tradegood_lists2blocks(cls, port_tradegood_list, price_dict, lang, groupby_parameter_type):
        logger = HenriqueLogger.func_level2logger(cls.port_tradegood_lists2blocks, logging.DEBUG)
        logger.debug({"port_tradegood_list": port_tradegood_list})

        if groupby_parameter_type == PriceSkillParameter.Type.PORTLIKE:
            from henrique.main.skill.price.by_port.price_by_port import PriceByPort

            blocks = [PriceByPort.port2text(port_codename, lmap(ig(1), l), price_dict, lang)
                      for port_codename, l in gb_tree_global(port_tradegood_list, [ig(0)])]
            return blocks

        if groupby_parameter_type == PriceSkillParameter.Type.TRADEGOOD:
            from henrique.main.skill.price.by_tradegood.price_by_tradegood import PriceByTradegood

            blocks = [PriceByTradegood.tradegood2text(tg_codename, lmap(ig(0), l), price_dict, lang)
                      for tg_codename, l in gb_tree_global(port_tradegood_list, [ig(1)])]
            return blocks

        raise Exception(groupby_parameter_type)

    @classmethod
    def packet2response(cls, packet):
        blocks = cls.packet2rowsblocks(packet)
        return Rowsblock.blocks2text(blocks)

    @classmethod
    @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2))
    def pattern_rate_trend(cls):
        # rstr_idk = RegexTool.rstr_iter2or(map(re.escape, cls.dict_lang2text_idk().values()))

        rstr_arrows = RegexTool.rstr_iter2or(map(re.escape, Trend.dict_trend2arrow().values()))
        rstr_rate_trend = RegexTool.join(r"", [r"\d{2,3}", rstr_arrows])

        # rstr = r"{}\s*$".format(RegexTool.rstr_iter2or([rstr_idk, rstr_rate_trend]))
        # rstr = r"{}\s*$".format(rstr_rate_trend)

        # raise Exception(rstr)
        pattern = re.compile(RegexTool.rstr2wordbounded(rstr_rate_trend), re.I)
        return pattern

    @classmethod
    def blocks2norm_list_for_unittest(cls, blocks):
        def row2header(row):
            match_list = list(cls.pattern_rate_trend().finditer(row))
            match = l_singleton2obj(match_list)
            s,e = MatchTool.match2span(match)
            return str2strip(row[:s])
            # return str2strip(cls.pattern_rate_trend().sub("", row))

        def block2norm_for_unittest(block):
            rows = block.splitlines()
            title = rows[0]

            row_header_list = lmap(row2header, rows[1:])
            return title, row_header_list

        return lmap(block2norm_for_unittest, blocks)

    @classmethod
    def blocks2norm_set_for_unittest(cls, blocks):
        norm_list = cls.blocks2norm_list_for_unittest(blocks)

        return [(title, set(row_header_list))
                for title, row_header_list in norm_list]

# WARMER.warmup()


from snownlp import SnowNLP
from snownlp import seg
from snownlp import sentiment
import numpy as np
import csv
import bottleneck as bn
import pandas as pd

path = "../utils/jd.csv"
# path = "../data/test_set.csv"

def test():

    s = SnowNLP(u'屏幕大小很喜欢，清晰度和色彩三星的屏太棒了，一直都用三星手机系统也很上手，从note8换成s21u有种飞的感觉，不愧为一代机皇，谢谢！')
    print(s.sentiments)
    # 文章关键字提取
    text = u'''
    我是一只猪，没有人比我厉害，我是一只猫，最漂亮的猫，我爱吃水果，我爱吃土豆，我喜欢睡大床
    '''
    s = SnowNLP(text)
    s.keywords(4)#提取关键词
    print(s.keywords(4))
    print(s.summary())

def train_snownlp():
    # train
    sentiment.train('../data/neg.csv',
                    '../data/pos.csv')  # 注意路径斜线别写错
    sentiment.save('sentiment1.marshal')
    # test
    q = SnowNLP(u'宝贝真的很不错，，这是我第二次买了，我的朋友都好喜欢，穿上特别漂亮！好性感！质量好好！大爱！')
    print(q.sentiments)

def predict_data(path):
    pre_dic = []
    neg_sample = {}
    neg_score = {}
    pos_sample = {}
    pos_score = {}
    with open(path, 'rt', encoding='GBK') as csvfile:
            reader = csv.reader(csvfile)
            for row in reader:

                q = SnowNLP(str(row))
                pre = q.sentiments
                pre_dic.append(pre)
                # print(pre,str(row))
    # with open(path,'r') as f:
    #     red = csv.DictReader(f)     # 通过csv自带的DictReader方法将数据转换成字典表
    #     for row in red:
    #         q = SnowNLP(str(row))
    #         pre = q.sentiments
    #         pre_dic.append(pre)
    #         print(str(row),pre)
    res = np.array(pre_dic)
    mean = np.mean(res)
    N = 3
    pos = -bn.partition(-res, N)[:N]
    # print(pos)
    # print(pos.shape)
    pos_index = bn.argpartition(-res,N)[:N]
    # print(pos_index)
    neg = bn.partition(res,N)[:N]
    # print(neg,neg.shape)
    neg_index = bn.argpartition(res, N)[:N]
    # print(neg_index)
    comment_info = pd.read_csv(path,encoding='GBK')

    pos_sample[0] = comment_info.loc[pos_index[0]-2]['content']  # 最大值
    pos_sample[1] = comment_info.loc[pos_index[1]-2]['content']
    pos_sample[2] = comment_info.loc[pos_index[2]-2]['content']
    neg_sample[0] = comment_info.loc[neg_index[0]-2]['content']  # 最小值
    neg_sample[1] = comment_info.loc[neg_index[1]-2]['content']
    neg_sample[2] = comment_info.loc[neg_index[2]-2]['content']
    print(pos_sample)
    print(neg_sample)
    return pos,pos_sample,neg,neg_sample,mean

# #
# predict_data(path)
# # test()


import json
import requests

from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE'
}

def get_all_teams():
    """
    This function will get all teams from the website.
    :return:
    """
    leagues_base_url = "https://www.fifaratings.com/leagues/"
    teams = []

    response = requests.get(leagues_base_url, headers=headers)
    soup = BeautifulSoup(response.content, "lxml")
    team_table = soup.find(
        "table", {"class": "table table-sm table-striped nowrap mb-0"})
    team_links = team_table.find_all("a", href=True)

    for team_link in team_links:
        if "/league/" in team_link["href"]:
            teams.append(team_link["href"])

    return teams

def get_team_info(team_url: str):
    """
    This function will get all team info from a team.
    :param team_url:
    :return:
    """
    team_data = []
    html_content = requests.get(team_url, headers=headers)

    soup = BeautifulSoup(html_content.content, "lxml")
    team_table = soup.find(
        "table", {"class": "table table-striped table-sm table-hover mb-0"})
    team_infos = team_table.tbody.find_all("tr")

    for team_info in team_infos:
        try:
            team_tds = team_info.find_all("td")

            if len(team_tds) > 3:
                team_url = team_tds[1].find("a")["href"]
                team_name = team_tds[1].text.strip().split(" ")
                team_name = team_name[0] + " " + team_name[1]
                team_points = team_tds[2].text.strip()

                team_data.append({
                    "url": team_url,
                    "name": team_name,
                    "points": team_points
                })
        except:
            pass

    return team_data

def get_team_players(data: dict):
    """
    This function will get all players from a team.
    :param data:
    :return:
    """
    html_content = requests.get(data.get('url'), headers=headers)

    members = []
    soup = BeautifulSoup(html_content.content, "lxml")
    player_table = soup.find(
        "table", attrs={"class": "table table-striped table-sm table-hover mb-0"})
    team_members_tr = player_table.tbody.find_all("tr")

    for team_member in team_members_tr:
        try:
            team_member_td = team_member.find_all("td")
            player = team_member_td[1].text.strip().split(" ")
            player_name = player[0] + " " + player[1] + \
                " " + player[2] + " " + player[3]
            player_overall_points = team_member_td[2].text.strip()
            player_potential_points = team_member_td[3].text.strip()

            member = {
                "name": player_name.strip(),
                "Overall": player_overall_points,
                "Potential": player_potential_points
            }
            members.append(member)
        except:
            pass

    data["players"] = members

    return data

def main():
    all_teams = []
    players = []
    teams = get_all_teams()

    for team in teams:
        team_infos = get_team_info(team)

        for team_info in team_infos:
            data = get_team_players(team_info)
            all_teams.append(data)
            players += data.get("players")

    with open("players.json", "w") as f:
        json.dump(players, f)

    with open("data.json", "w") as f:
        json.dump(all_teams, f)

if __name__ == "__main__":
    main()