# POST requests
# I created a little web service on the university server that you can post to
# Endpoint: http://2id60.win.tue.nl:3000
# Posts format: {"userId": 1, "id": 1, "title": "lorem", "body": "ipsum"}

import urllib.request
import urllib.parse

url = 'http://2id60.win.tue.nl:3000/posts'
values = {
	'userId': '1',
	'id': '4',
	'title': 'My first Post!',
	'body': 'My first Body!'
}
data = urllib.parse.urlencode(values).encode("utf-8") # Encode JSON
req = urllib.request.Request(url, data) # Create POST request
response = urllib.request.urlopen(req) # Do the request
print(response.read())

# Check it by downloading it again
import json
import pprint
u = urllib.request.urlopen(url+'?userId=1')
posts = u.read().decode('utf8')
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(posts)


#-*- coding: utf-8 -*-
import cv2
import numpy as np
from matplotlib import pyplot as plt

def matplotlib():
	img = cv2.imread('falta.jpeg')
	plt.imshow(img)
	plt.xticks([]), plt.yticks([])
	plt.show()
	return

#img = cv2.imread("fuga.jpg")
#Para escala de grises
pic = ['dificultadRespirar.jpg','dolorGarganta.jpg','dolorToraxRespirar.jpg','resequedadGarganta.jpg','respirarEjercicio.jpg','ruidosRespiratorios.jpg','tos.jpg' ,'tosCronica.jpg','tosSeca.jpg'];
picName = ['dificultadRespirarGray.jpg','dolorGargantaGray.jpg','dolorToraxRespirarGray.jpg','resequedadGargantaGray.jpg','respirarEjercicioGray.jpg','ruidosRespiratoriosGray.jpg','tosGray.jpg','tosCronicaGray.jpg','tosSecaGray.jpg'];

for i in range(0,9):
	img = cv2.imread(pic[i], cv2.IMREAD_GRAYSCALE)
	#Para guardar una imagen
	cv2.imwrite(picName[i],img)
'''
img1 = "dolhom.png"
img = cv2.imread(img1, cv2.IMREAD_GRAYSCALE)
#Para guardar una imagen
cv2.imwrite("dolhomGray.png",img)
#cv2.imshow('fuga', img)
#Espera a recibir una recla
#cv2.waitKey(0)
#te ayuda para cerrar las ventanas que se abran en el programa
'''
cv2.destroyAllWindows()

#matplotlib()


# -*- coding: utf-8 -*-
from scrapy import Spider, Request
from pyquery import PyQuery as pq
import re
from Xav.items import XavItem

class XavSpider(Spider):
    name = 'xav'
    allowed_domains = ['xcaoav.com']
    start_urls = ['https://xcaoav.com/forum-1.htm']

    def parse(self, response):
        res = re.findall('forum-(\d)-(.*?)-tid.htm">... (.*?)</a>', response.text)
        category = res[0][0]
        total_page = res[0][2]
        for page in range(1, int(total_page)+1):
      #  for page in range(1, 100):
            page_url = 'https://xcaoav.com/forum-%s-%s-tid.htm' % (str(category), str(page))

            yield Request(url=page_url, callback=self.parse_page, meta={'category': category})

        category_list = [1,2,8,9]
        for category in category_list:
            category_url = 'https://xcaoav.com/forum-%s.htm' % str(category)

            yield Request(url=category_url, callback=self.parse)

    def parse_page(self, response):
        category = response.meta['category']
        pattern = re.compile('class="thread thread_agrees_0 " href="(.*?)"', re.S)
        uris = re.findall(pattern, response.text)
        for uri in uris:
            link = 'https://xcaoav.com/%s' % uri

            yield Request(url=link, callback=self.parse_detail, meta={'category': category})

    def parse_detail(self, response):
        category = response.meta['category']
        img_url_list = []
        doc = pq(response.text)
        title = doc('h3').eq(0).text()
        post_time = doc.find('.light').eq(1).text()
        detail = doc.find('#thread_post div').text()
        torrent_url = re.findall('下載地址：(.*?)\n', detail)[0]
        img_list = doc('#thread_post img').items()
        for i in img_list:
            img = i.attr.src
            img_url_list.append(img)

        yield XavItem(title=title, category=category, post_time=post_time, torrent_url=torrent_url, detail=detail, image_urls=img_url_list)


import pandas as pd

df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data', header=None)

from sklearn.model_selection import train_test_split

X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,
                                                    stratify=y, random_state=0)

from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)
X_test_std = sc.transform(X_test)

'''
import numpy as np

cov_mat = np.cov(X_train_std.T)

eigen_vals, eigen_vecs = np.linalg.eig(cov_mat)

eigen_pairs = [(np.abs(eigen_vals[i]), eigen_vecs[:, i]) for i in range(len(eigen_vals))]

eigen_pairs.sort(key=lambda k: k[0], reverse=True)

w = np.hstack((eigen_pairs[0][1][:, np.newaxis], eigen_pairs[1][1][:, np.newaxis]))

X_train_pca = X_train_std.dot(w)
'''

from data_prep import plot_decisions_regions

from sklearn.linear_model import LogisticRegression
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

pca = PCA(n_components=2)
lr = LogisticRegression()

X_train_pca = pca.fit_transform(X_train_std)
X_test_pca = pca.transform(X_test_std)

lr.fit(X_train_pca, y_train)
plot_decisions_regions(X_train_pca, y_train, classifier=lr)
plt.xlabel('PC 1')
plt.ylabel('PC 2')
plt.legend(loc='lower left')
plt.show()


import datetime
from datetime import date
from Model.UserInputs import *
import numpy as np

# black scholes variables
r = 0.01
T = 0
S = df["Adj Close"][-1]
K = []
sigma = []
current_prices = []
open_interest = []
value_type = []

class CleaningData:

    @staticmethod
    def callOrPut():
        global contract_type
        if 'c' in option_type.lower():
            contract_type = chain['calls']
        else:
            contract_type = chain['puts']

    @staticmethod
    def gettingStrike():
        global K
        vector = np.vectorize(np.float)
        K = list(vector(contract_type[['Strike']].values))
        K = [float(x) for x in K]

    @staticmethod
    def gettingLastPrice():
        global current_prices
        vector = np.vectorize(np.float)
        current_prices = list(vector(contract_type[['Last Price']].values))
        current_prices = [float(x) for x in current_prices]

    @staticmethod
    def gettingSigma():
        "this is getting implied volatility"
        global sigma
        new_sig = []
        sigma = list(contract_type[['Implied Volatility']].values)

        for i in sigma:
            iv = float(i[-1][:-2]) / 100
            new_sig.append(iv)
        sigma = new_sig

    @staticmethod
    def gettingT():
        global T
        contract_name = str(contract_type[['Contract Name']].values[0][0])
        date_str = contract_name[len(stock):]
        year = 2000 + int(date_str[:2])
        month = int(date_str[2:4])
        day = int(date_str[4:6])

        today = datetime.today().date()
        expiry_date = date(year, month, day)
        diff = expiry_date - today
        T = diff.days / 365

    @staticmethod
    def getOpenInterest():
        global open_interest
        open_interest = list(contract_type['Open Interest'])

    @staticmethod
    def getValueType():
        for strike in K:
            if strike >= S:
                value_type.append("---Out---")
            else:
                value_type.append("***In***")

    @staticmethod
    def printers():
        print(f'Strike Prices: {K}\n',
              f'Implied Volatility: {sigma}\n',
              f'Value Type: {value_type}')

CleaningData.callOrPut()
CleaningData.gettingT()
CleaningData.gettingStrike()
CleaningData.gettingSigma()
CleaningData.gettingLastPrice()
CleaningData.getValueType()


r"""
Gives several definitions used in the import / analysis process
"""

from protocol import Protocol

#
# The location of files
#
BASE_PATH = "/Data" # the files are stored in the parent directory

#
# DATA INDICES INTO FILES.PY
#
PATH = 0   # the path to the data file
CAP = 1    # the capacitance of the cell
O_P = 2    # the value of optical power in DACs
O_T = 3    # the value of the optical power threshold for this particular cell
E_P = 4    # the value of injected electrical current in pA
E_T = 5    # the value of the electrical threshold for this particular cell
APS = 6    # true if action potentials were visually evident during recording
CEL = 7    # the cell number
PAR = 8    # the offset data set, 1 = 1ms, 2 = 2ms, 3 = 3ms shift in electrical onset

#
# Data indices into the CELLS object in files.py
#
C_CAP = 0   # The cell capacitance
C_OPT = 1   # The cell optical threshold (DACs)
C_ELC = 2   # The cell electrical threshold (pA)

# ID, First offset, num offsets, offset delta
PARAMS = [
    Protocol(0, -5, 18, 1),
    Protocol(1, -5, 18, 2, column_count=2),
    Protocol(2, -5, 18, 3, column_count=2),
    Protocol(3, -5, 18, 2, column_count=2, optical_onset=1.5),
    Protocol(4, -5, 18, 2, column_count=1, optical_onset=1.5),
    Protocol(5, 10, 9, 4, column_count=2, optical_onset=1.5),
    Protocol(6, -3, 9, 4, column_count=2, optical_onset=1.5),
]

JNC_POT = -12.8 / 1000.


import numpy as np

class resultados_estrategias():
    def __init__(self, accion_sin_evaluar,estrategia="NO"):
        self.accion = accion_sin_evaluar
        self.resultado = {}
        self.estrategia=estrategia

    def evalua_resultados(self):
        self.accion['Retornos'] = np.where(((self.accion.Trade != 0)), self.accion.Dif_Close - 0.0085,
                                           np.where(self.accion.Posicion == 1, self.accion.Dif_Close, 0))
        self.accion['capital_estrategia'] = (
            self.accion.Retornos + 1).cumprod() * 100
        self.accion['Buy_and_Hold'] = (
            self.accion.Dif_Close + 1).cumprod() * 100
        self.accion_evaluada = self.accion.copy(
        )
        self.resultado['Beneficio_Bruto'] = round(
            self.accion_evaluada.iloc[-1]['capital_estrategia']-100, 2)

        dataframe_estrategia = self.accion_evaluada[self.accion.Trade != 0]
        self.resultado['trades'] = self.accion_evaluada['Trade'].count()
        self.accion_evaluada = self.accion_evaluada[[
            'Posicion', 'capital_estrategia']]

        self.accion_evaluada['resultado_trade'] = np.where((self.accion_evaluada.Posicion == -1) & (self.accion_evaluada.Posicion.shift(
        ) == 1), self.accion_evaluada['capital_estrategia']-dataframe_estrategia['capital_estrategia'].shift(), 0)
        self.resultado['trades_ganadores'] = self.accion_evaluada.resultado_trade[self.accion_evaluada.resultado_trade >= 0].count()
        self.resultado['suma_trades_ganadores'] = round(
            self.accion_evaluada.resultado_trade[self.accion_evaluada.resultado_trade >= 0].sum(), 2)
        self.resultado['trades_perdedores'] = self.accion_evaluada.resultado_trade[self.accion_evaluada.resultado_trade < 0].count()
        self.resultado['suma_trades_perdedores'] = abs(round(
            self.accion_evaluada.resultado_trade[self.accion_evaluada.resultado_trade < 0].sum(), 2))

        if self.resultado['trades_ganadores'] == 0:
            self.resultado['Ratio Acierto'] = 0
            self.resultado['Profit Factor'] = 0

        elif self.resultado['trades_perdedores'] == 0:
            self.resultado['Ratio Acierto'] = 100
            self.resultado['Profit Factor'] = 100

        else:
            self.resultado['Ratio Acierto'] = round(
                100 * self.resultado['trades_ganadores'] / self.resultado['trades'], 2)
            self.resultado['Profit Factor'] = round(
                self.resultado['suma_trades_ganadores'] / self.resultado['suma_trades_perdedores'], 2)

        self.resultado['capital_estrategia'] = round(
            self.accion['capital_estrategia'].iloc[-1], 2)
        self.resultado['Buy_Hold'] = round(
            self.accion['Buy_and_Hold'].iloc[-1], 2)
        if self.estrategia != "NO":
            self.resultado["Estrategia"]=self.estrategia
        else:
            pass

        return self.resultado, self.accion


import os
import sys
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from sklearn.model_selection import train_test_split
import torch
TEST_SET_DIR = '/scratch/sr365/ML_MM_Benchmark/testsets'

def get_data_into_loaders(data_x, data_y, batch_size, DataSetClass, rand_seed=0, test_ratio=0.3):
    """
    Helper function that takes structured data_x and data_y into dataloaders
    :param data_x: the structured x data
    :param data_y: the structured y data
    :param rand_seed: the random seed
    :param test_ratio: The testing ratio
    :return: train_loader, test_loader: The pytorch data loader file
    """
    if test_ratio == 1:         # Test case
        print('This is testing mode!!! Test ratio = 1')
        test_data = DataSetClass(data_x, data_y)
        test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)
        return None, test_loader

    x_train, x_test, y_train, y_test = train_test_split(data_x, data_y, test_size=test_ratio,
                                                        random_state=rand_seed)
    print('total number of training sample is {}, the dimension of the feature is {}'.format(len(x_train), len(x_train[0])))
    print('total number of test sample is {}'.format(len(y_test)))

    # Construct the dataset using a outside class
    train_data = DataSetClass(x_train, y_train)
    test_data = DataSetClass(x_test, y_test)

    # Construct train_loader and test_loader
    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size)
    test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)

    return train_loader, test_loader

def normalize_np(x, x_max_list=None, x_min_list=None):
    """
    Normalize the x into [-1, 1] range in each dimension [:, i]
    :param x: np array to be normalized
    :return: normalized np array
    """
    if x_max_list is not None:
        if x_min_list is None or len(x[0]) != len(x_max_list) or len(x_max_list) != len(x_min_list):
            print("In normalize_np, your dimension does not match with provided x_max, try again")
            quit()

    new_x_max_list = []
    new_x_min_list = []
    for i in range(len(x[0])):
        if x_max_list is None:
            x_max = np.max(x[:, i])
            x_min = np.min(x[:, i])
        else:
            x_max = x_max_list[i]
            x_min = x_min_list[i]
        x_range = (x_max - x_min ) /2.
        x_avg = (x_max + x_min) / 2.
        x[:, i] = (x[:, i] - x_avg) / x_range
        print("In normalize_np, row ", str(i), " your max is:", np.max(x[:, i]))
        print("In normalize_np, row ", str(i), " your min is:", np.min(x[:, i]))
        if x_max_list is None:
            assert np.max(x[:, i]) - 1 < 0.0001, 'your normalization is wrong'
            assert np.min(x[:, i]) + 1 < 0.0001, 'your normalization is wrong'
            new_x_max_list.append(x_max)
            new_x_min_list.append(x_min)
    return x, np.array(new_x_max_list), np.array(new_x_min_list)

def read_data_color_filter(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Color')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values

    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Color')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        return get_data_into_loaders(data_x, data_y, flags.batch_size,  SimulatedDataSet_regress, rand_seed=flags.rand_seed,test_ratio=1)

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data_Yang(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Yang')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values
    # Normalize the dataset
    data_x, x_max, x_min = normalize_np(data_x)

    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Yang')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        data_x, _, _, = normalize_np(data_x, x_max, x_min)
        print('This is Yang dataset with data_x shape of', np.shape(data_x))
        return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=1)

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data_peurifoy(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Peurifoy')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values
    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Peurifoy')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        data_x = (data_x - 50) / 20.
        print('This is Perifoy dataset with data_x shape of', np.shape(data_x))
        return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed,test_ratio=1)

    # The geometric boundary of peurifoy dataset is [30, 70], normalizing manually
    data_x = (data_x - 50) / 20.

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data(flags, eval_data_all=False):
    """
    The data reader allocator function
    The input is categorized into couple of different possibilities
    0. meta_material
    1. gaussian_mixture
    2. sine_wave
    3. naval_propulsion
    4. robotic_arm
    5. ballistics
    :param flags: The input flag of the input data set
    :param eval_data_all: The switch to turn on if you want to put all data in evaluation data
    :return:
    """
    print("In read_data, flags.data_set =", flags.data_set)
    if 'Yang' in flags.data_set or 'ADM' in flags.data_set:
        train_loader, test_loader = read_data_Yang(flags,eval_data_all=eval_data_all)
    elif 'Peurifoy' in  flags.data_set :
        train_loader, test_loader = read_data_peurifoy(flags,eval_data_all=eval_data_all)
    elif 'olor' in flags.data_set:
        train_loader, test_loader =read_data_color_filter(flags,eval_data_all=eval_data_all)
    else:
        sys.exit("Your flags.data_set entry is not correct, check again!")
    return train_loader, test_loader

class MetaMaterialDataSet(Dataset):
    """ The Meta Material Dataset Class """
    def __init__(self, ftr, lbl, bool_train):
        """
        Instantiate the Dataset Object
        :param ftr: the features which is always the Geometry !!
        :param lbl: the labels, which is always the Spectra !!
        :param bool_train:
        """
        self.ftr = ftr
        self.lbl = lbl
        self.bool_train = bool_train
        self.len = len(ftr)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.ftr[ind, :], self.lbl[ind, :]

class SimulatedDataSet_class_1d_to_1d(Dataset):
    """ The simulated Dataset Class for classification purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind], self.y[ind]

class SimulatedDataSet_class(Dataset):
    """ The simulated Dataset Class for classification purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind, :], self.y[ind]

class SimulatedDataSet_regress(Dataset):
    """ The simulated Dataset Class for regression purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind, :], self.y[ind, :]


def make_album():
    name = input('\tВведите название альбома: ')
    singer = input('\tВведите имя исполнителя: ')
    quantity = input('\tВведите количество песен: ')
    album = {}
    album['album_name'] = name
    album['singer_name'] = singer
    album['number_of_songs'] = quantity
    return album

def show_collections(collections_list):
    print('\nУ вас в коллекции:')
    for album in collections_list:
        print('\tНазвание альбома: ' + album['album_name'].upper())
        print('\tАвтор альбома: ' + album['singer_name'].title())
        print('\tКоличество песен: ' + album['singer_name'] + '\n')
    return

music_collections = []

while True:#запрос информации от пользователя
    print('\nДавайте создадим вашу музыкальную коллекцию!')
    answer = input('\tПродолжим? Y\\N:')
    if answer.lower() == 'n':
        print('\t\t Quit')
        break
    music_collections.append(make_album())

show_collections(music_collections)


# -*- coding:utf8 -*-

import time

from assertpy import assert_that as asserts

from common import sqlbase, page
from common.base import log, Base
from common.interface import createCustomer, createApartmentContract, addHouseContractAndFitment
from contract.houseContract.page import houseContractPage

@log
def test_1044():
    """修改委托合同租金"""

    # describe：在委托合同详情中，修改业绩核算周期对应时间段的委托合同租金策略中的租金，业绩中对应发生变化
    # data：1、业绩审核状态为待审核；2、委托合同状态为待审核或者已初审；3、提前记录原业绩中的核算收进价和差价业绩；
    # result：1、新产生的业绩中核算收进价与原记录不同；2、新的差价业绩与原来的值不同；

    fileName = 'apartmentAchievement_1044'

    with Base() as base:
        # 创建委托合同和出租合同
        houseSql = sqlbase.serach(
            "select house_id,residential_id,building_id,house_code from house where deleted=0 and city_code=330100 order by rand() limit 1")  # 获取随机开发房源
        houseInfo = {'houseID': houseSql[0], 'residentialID': houseSql[1], 'buildingID': houseSql[2],'houseCode': houseSql[3]}
        dateInfo = sqlbase.serach("select date(sysdate()),date_add(date(sysdate()),INTERVAL 1 day),date_add(date(sysdate()),interval 2 year),date_add(date(sysdate()),interval 27 month),"
                                  "date_add(date(sysdate()),INTERVAL 1 month),date_add(date(sysdate()),INTERVAL 6 month) from dual")  # 日期元素
        apartmentId = addHouseContractAndFitment(apartment_type='MANAGE', entrust_type='SHARE', sign_date=dateInfo[0],
                                                 owner_sign_date=dateInfo[0], entrust_start_date=dateInfo[0],
                                                 entrust_end_date=dateInfo[2], delay_date=dateInfo[3],
                                                 free_start_date=dateInfo[0], free_end_date=dateInfo[4],
                                                 first_pay_date=dateInfo[0], second_pay_date=dateInfo[4],
                                                 rent=3000, parking=100, year_service_fee=500, payment_cycle='MONTH',
                                                 fitment_start_date=dateInfo[0], fitment_end_date=dateInfo[4], rooms=3,
                                                 fitmentCost=88888,houseInfo=houseInfo)
        rentPriceInfo = sqlbase.serach("select rent_price,date(sysdate()) from apartment where apartment_id='%s'" % apartmentId)
        rentPrice = float(rentPriceInfo[0])
        customer = createCustomer()
        apartmentContractInfo = createApartmentContract(apartement_id=apartmentId, customerInfo=customer,
                                                        rent_price=rentPrice, sign_date=dateInfo[0],
                                                        rent_start_date=dateInfo[1], rent_end_date=dateInfo[5],  # 承租6个月
                                                        deposit=rentPrice, payment_cycle='MONTH')
        apartmentContractNum = apartmentContractInfo['contractNum']
        achievementsql = "select substring_index(aca.house_code,'-',1),aca.is_active,aca.audit_status,aca.contract_audit_status,aca.profits_fee,aca.rent_cost from apartment_contract_achievement aca " \
                          "inner join apartment a on a.apartment_code=aca.house_code and a.apartment_id='%s' where aca.contract_num='%s' and aca.deleted=0" % (
                          apartmentId, apartmentContractNum)
        base.diffAssert(lambda test: asserts(sqlbase.waitData(achievementsql, 1)).is_true(), 1057,
                        u'%s:合同 %s 对应业绩生成异常' % (fileName, apartmentContractNum))
        profits_feeOld = sqlbase.serach(achievementsql)[4]  # 差价业绩
        rent_cost_old = sqlbase.serach(achievementsql)[5]  # 核算收进价
        houseCode = sqlbase.serach(achievementsql)[0]
        # 修改租金策略并且复审
        base.open(page.entrustContractPage, houseContractPage.contractSearchMould['tr_contract'])
        # base.input_text(houseContractPage.contractSearchMould['contract_num_loc'], info[2])
        base.input_text(houseContractPage.contractSearchMould['residential_name_loc'], houseCode)
        base.click(houseContractPage.contractSearchMould['search_button_loc'])  # 搜索
        base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待列表刷新
        base.dblclick(houseContractPage.contractSearchMould['tr_contract'],
                      checkLoc=houseContractPage.addHouseContractMould['contract_num_loc'])  # 双击第一条数据
        base.click(houseContractPage.addHouseContractMould['contract_strategy1.1_loc'])
        base.input_text(houseContractPage.addHouseContractMould['contract_strategy1_loc'], 4000)  # 第一年租金策略
        base.click(houseContractPage.addHouseContractMould['page1_save_button'])  # 保存
        for i in range(3):
            try:
                base.check_submit()
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['rent_save_button'])  # 保存
                    base.check_submit()
        # 委托合同复审
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=1)  # 租金页面
        base.click(houseContractPage.addHouseContractMould['rent_detail_selectAll'])  # 全选
        base.click(houseContractPage.addHouseContractMould['rent_audit_loc'])  # 审核
        base.click(houseContractPage.addHouseContractMould['audit_pass_loc'])  # 通过
        base.click(houseContractPage.addHouseContractMould['rent_audit_confirm'])  # 确认
        # 初审
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=3)  # 最后一页
        time.sleep(2)
        base.script('$("button[status=\'PASS\']")[2].click()')  # 初审
        base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
        for i in range(3):
            try:
                base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待数据刷新
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
                    base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待数据刷新
        # 复审
        base.dblclick(houseContractPage.contractSearchMould['tr_contract'],
                      checkLoc=houseContractPage.addHouseContractMould['contract_num_loc'])  # 双击第一条数据
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=3)  # 最后一页
        time.sleep(2)
        base.script('$("button[status=\'APPROVED\']")[1].click()')  # 复审
        base.click(houseContractPage.addHouseContractMould['rentdif_cofirm_loc'])  # 租金策略不同提示确定
        base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
        for i in range(3):
            try:
                base.check_submit()
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
                    base.check_submit()
        #获取最新业绩信息
        time.sleep(10)
        profits_feeNew = sqlbase.serach(achievementsql)[4]
        rent_cost_new = sqlbase.serach(achievementsql)[5]
        base.diffAssert(lambda test:asserts(rent_cost_new).is_not_equal_to(rent_cost_old),1044,
                        u'%s:出租合同 %s 对应委托成本修改后业绩中核算收进价异常，修改前 %s 修改后 %s' % (fileName, apartmentContractNum, rent_cost_old, rent_cost_new))
        base.diffAssert(lambda test: asserts(profits_feeNew).is_not_equal_to(profits_feeOld),1044,
                        u'%s:出租合同 %s 对应委托成本修改后业绩中差价业绩异常，修改前 %s 修改后 %s' % (fileName, apartmentContractNum,profits_feeOld, profits_feeNew))

test_1044()


# -*- coding: utf-8 -*-
from numpy import *
import csv
import classify
if __name__ == '__main__':

    path='/mnt/'
    f1=open(path+"train.txt","w")
    f2=open(path+"test.txt","w")
    with open(path+'image_class.csv','rb') as Label1:
        reader = csv.reader(Label1)
        labelArr1 = []
        for row in reader:
            imgpath=row[3]
            cloth=row[4]
            label=-1
            if cloth=='衬衫':
                label=0
            elif cloth=='T恤' or cloth=='POLO衫':
                label=1
            elif cloth=='背心':
                label=2
            elif cloth=='雪纺衫' or cloth=='针织衫':
                label=3
            elif cloth=='连衣裙':
                label=4
            elif cloth=='半身裙':
                label=5
            elif cloth=='牛仔裤':
                label=6
            elif cloth=='休闲裤':
                label=7
            elif cloth=='短裤':
                label=8
            else:
                continue
            classify.findBody(img)

            if(random.random()<0.9):
                f1.write(imgpath+" "+str(label)+"\n")
            else:
                f2.write(imgpath+" "+str(label)+"\n")


#!/usr/bin/env python
import argparse
import bugsy
import config
import datetime
import github
import json
import logging
import mozreview
import os
import psycopg2
import sys
import time
import traceback

sys.path.insert(0, os.path.normpath(os.path.join(os.path.normpath(
                os.path.abspath(os.path.dirname(__file__))), '..', '..',
                'pylib', 'mozautomation')))
from mozautomation.commitparser import parse_bugs
from mozlog.structured import commandline

import transplant
import treestatus

GITHUB_COMMENT_LIMIT = 10  # max updates to post to github / iteration
MOZREVIEW_COMMENT_LIMIT = 10  # max updates to post to reviewboard / iteration

def handle_pending_mozreview_pullrequests(logger, dbconn):
    gh = github.connect()
    if not gh:
        return

    bzurl = config.get('bugzilla')['url']

    cursor = dbconn.cursor()

    query = """
        select id,ghuser,repo,pullrequest,destination,bzuserid,bzcookie,bugid,
               pingback_url
        from MozreviewPullRequest
        where landed is null
    """
    cursor.execute(query)

    landed_revisions = []
    mozreview_updates = []
    for row in cursor.fetchall():
        (transplant_id, ghuser, repo, pullrequest, destination, bzuserid,
         bzcookie, bugid, pingback_url) = row

        logger.info('attempting to import pullrequest: %s' % transplant_id)

        # see if we can extract the bug from the commit message
        if bugid is None:
            title, body = github.retrieve_issue(gh, ghuser, repo, pullrequest)
            bugs = parse_bugs(title)
            if bugs:
                bugid = bugs[0]
                logger.info('using bug %s from issue title' % bugid)
                landed_revisions.append([bugid, None, None, transplant_id])

        # still no luck, attempt to autofile a bug on the user's behalf
        if bugid is None:
            logger.info('attempting to autofile bug for: %s' % transplant_id)

            b = bugsy.Bugsy(userid=bzuserid, cookie=bzcookie,
                            bugzilla_url=bzurl)
            if not b:
                logger.info('could not connect to bugzilla instance at %s for '
                            'pullrequest id %s' % (bzurl, transplant_id))
                error = 'could not connect to bugzilla. bad credentials?'
            else:
                bug = bugsy.Bug()

                # Summary is required, the rest have defaults or are optional
                bug.summary = title

                if config.testing():
                    bug.product = 'TestProduct'
                    bug.component = 'TestComponent'
                else:
                    # TODO: determine a better product & component than the
                    # defaults provided by Bugsy
                    pass

                pr_url = github.url_for_pullrequest(ghuser,repo, pullrequest)
                bug.add_comment('%s\n\nImported from: %s' % (body, pr_url))

                try:
                    b.put(bug)
                    bugid = bug.id
                    logger.info('created bug: %s ' % bugid)
                    landed_revisions.append([bugid, None, None, transplant_id])
                except bugsy.BugsyException as e:
                    logger.info('transplant failed: could not create new bug: %s '
                                % e.msg)
                    landed_revisions.append([None, False, e.msg, transplant_id])

                    # set up data to be posted back to mozreview
                    data = {
                        'request_id': transplant_id,
                        'bugid': None,
                        'landed': False,
                        'error_msg': 'could not create new bug: ' + e.msg,
                        'result': ''
                    }

                    mozreview_updates.append([transplant_id, pingback_url,
                                              json.dumps(data)])

        landed, result = transplant.transplant_to_mozreview(gh, destination,
                                                            ghuser, repo,
                                                            pullrequest,
                                                            bzuserid, bzcookie,
                                                            bugid)

        if landed:
            logger.info(('transplanted from'
                         ' https://github.com/%s/%s/pull/%s'
                         ' to destination: %s new revision: %s') %
                        (ghuser, repo, pullrequest, destination, result))
        else:
            logger.info(('transplant failed'
                         ' https://github.com/%s/%s/pull/%s'
                         ' destination: %s error: %s') %
                        (ghuser, repo, pullrequest, destination, result))

        landed_revisions.append([bugid, landed, result, transplant_id])

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'bugid': bugid,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])

    if landed_revisions:
        query = """
            update MozreviewPullRequest set bugid=%s,landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, landed_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(request_id,pingback_url,data)
            values(%s,%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()

def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()

    query = """
        select id,tree,rev,destination,trysyntax,push_bookmark,pingback_url
        from Transplant
        where landed is null
    """
    cursor.execute(query)

    landed_revisions = []
    mozreview_updates = []
    for row in cursor.fetchall():
        (transplant_id, tree, rev, destination, trysyntax, push_bookmark,
            pingback_url) = row

        if not treestatus.tree_is_open(destination):
            continue

        landed, result = transplant.transplant(tree, destination, rev,
                                               trysyntax, push_bookmark)
        if landed:
            logger.info(('transplanted from tree: %s rev: %s'
                         ' to destination: %s new revision: %s') %
                        (tree, rev, destination, result))
        else:
            if 'is CLOSED!' in result:
                logger.info('transplant failed: tree: %s is closed - '
                             ' retrying later.' % tree)

                # continuing here will skip updating the autoland request
                # so we will attempt to land it again later.
                continue
            else:
                logger.info('transplant failed: tree: %s rev: %s '
                            'destination: %s error: %s' %
                            (tree, rev, destination, result))

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])

        landed_revisions.append([landed, result, transplant_id])

    if landed_revisions:
        query = """
            update Transplant set landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, landed_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(request_id,pingback_url,data)
            values(%s,%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()

def handle_pending_github_updates(logger, dbconn):
    """Attempt to post updates to github"""

    gh = github.connect()
    if not gh:
        return

    cursor = dbconn.cursor()
    query = """
        select id,ghuser,repo,pullrequest,destination,landed,result,pingback_url
        from MozreviewPullRequest
        where landed is not NULL and pullrequest_updated is NULL
        limit %(limit)s
    """
    cursor.execute(query, {'limit': GITHUB_COMMENT_LIMIT})

    updated = []
    for row in cursor.fetchall():
        ghuser = row[1]
        repo = row[2]
        pullrequest = row[3]
        landed = row[5]
        result = row[6]
        pingback_url = row[7]

        if landed:
            message = 'Your new review request is available at: %s' % result
        else:
            message = 'Something went wrong importing to mozreview: %s' % result

        logger.info('attempting to add comment to pullrequest for request: %s' % row[0])

        worked = github.add_issue_comment(gh, ghuser, repo, pullrequest,
                                          message)
        if worked:
            updated.append([row[0]])
        else:
            logger.info('commenting on pull request failed')

    if updated:
        query = """
            update MozreviewPullRequest set pullrequest_updated=TRUE
            where id=%s
        """
        cursor.executemany(query, updated)
        dbconn.commit()

def handle_pending_mozreview_updates(logger, dbconn):
    """Attempt to post updates to mozreview"""

    cursor = dbconn.cursor()
    query = """
        select request_id,pingback_url,data
        from MozreviewUpdate
        limit %(limit)s
    """
    cursor.execute(query, {'limit': MOZREVIEW_COMMENT_LIMIT})

    mozreview_auth = mozreview.read_credentials()

    updated = []
    for row in cursor.fetchall():
        request_id, pingback_url, data = row
        logger.info('trying to post mozreview update to: %s for request: %s' %
                    (pingback_url, request_id))

        status_code, text = mozreview.update_review(mozreview_auth,
                                                    pingback_url, data)
        if status_code == 200:
            updated.append([request_id])
        else:
            logger.info('failed: %s - %s' % (status_code, text))

    if updated:
        query = """
            delete from MozreviewUpdate
            where request_id=%s
        """
        cursor.executemany(query, updated)
        dbconn.commit()

def get_dbconn(dsn):
    dbconn = None
    while not dbconn:
        try:
            dbconn = psycopg2.connect(dsn)
        except psycopg2.OperationalError:
            time.sleep(0.1)
    return dbconn

def main():
    parser = argparse.ArgumentParser()

    dsn = config.get('database')

    parser.add_argument('--dsn', default=dsn,
                        help="Postgresql DSN connection string")
    commandline.add_logging_group(parser)
    args = parser.parse_args()

    logging.basicConfig()
    logger = commandline.setup_logging('autoland', vars(args), {})
    logger.info('starting autoland')

    dbconn = get_dbconn(args.dsn)
    last_error_msg = None
    while True:
        try:
            handle_pending_mozreview_pullrequests(logger, dbconn)
            handle_pending_transplants(logger, dbconn)
            handle_pending_github_updates(logger, dbconn)
            handle_pending_mozreview_updates(logger, dbconn)
            time.sleep(0.25)
        except KeyboardInterrupt:
            break
        except psycopg2.InterfaceError:
            dbconn = get_dbconn(args.dsn)
        except:
            # If things go really badly, we might see the same exception
            # thousands of times in a row. There's not really any point in
            # logging it more than once.
            error_msg = traceback.format_exc()
            if error_msg != last_error_msg:
                logger.error(error_msg)
                last_error_msg = error_msg

if __name__ == "__main__":
    main()


"""Module to calculate the Yacht game score."""

from collections import defaultdict

# Score categories.
# Change the values as you see fit.

# multiply number of die value by that value (e.g. [3, 3, 3, 1, 2] = 3 * 3 * 3 * 3 = 27)
ONES = 1
TWOS = 2
THREES = 3
FOURS = 4
FIVES = 5
SIXES = 6

# total of all the dice if triple + pair
FULL_HOUSE = 7

# total of four matching numbers
FOUR_OF_A_KIND = 8

# dice in [1, 2, 3, 4, 5] score 30
LITTLE_STRAIGHT = 9

# dice in [2, 3, 4, 5, 6] score 30
BIG_STRAIGHT = 10

# sum of dice
CHOICE = 11

# five-of-a-kind scores 50
YACHT = 12

def score(dice: list[int], category) -> int:
    """Calculate the score for the given series of dice.

    :param dice: list[int] - series of dice to calculate the score
    :param category: X - specific score category
    :return: int - score of dice based on category
    """

    # prep a dict of the dice elements
    dice_dict = defaultdict(int)
    for die in dice:
        dice_dict[die] += 1

    keys = len(dice_dict.keys())

    if category in [ONES, TWOS, THREES, FOURS, FIVES, SIXES]:
        return category * dice_dict.get(category, 0)

    if category == FULL_HOUSE:
        if keys == 2 and sorted(dice_dict.values()) == [2, 3]:
            return sum(dice)

    if category == FOUR_OF_A_KIND:
        for key, count in dice_dict.items():
            if count >= 4:
                return 4 * key

    if category == LITTLE_STRAIGHT:
        if sorted(dice) == [1, 2, 3, 4, 5]:
            return 30

    if category == BIG_STRAIGHT:
        if sorted(dice) == [2, 3, 4, 5, 6]:
            return 30

    if category == CHOICE:
        return sum(dice)

    if category == YACHT:
        if keys == 1 and sorted(dice_dict.values()) == [5]:
            return 50

    return 0


import argparse
import json
import os
# data: q, cq, (dq), (pq), y, *x, *cx
# shared: x, cx, (dx), (px), word_counter, char_counter, word2vec
# no metadata
from collections import Counter
import pdb
from tqdm import tqdm
import re
import logging
import numpy as np
import random
MAX_NUM=1e09

def main():
    import pdb
    #pdb.set_trace()
    args = get_args()
    source_train_f = os.path.join(args.source_dir,args.version,'raw','train.json')
    source_dev_f = os.path.join(args.source_dir,args.version,'raw','dev.json')
    train_data = prepro_each(source_train_f)
    dev_data = prepro_each(source_dev_f)
    word_2_id_dict, id_2_word_dict, pretrained_emb = build_dict(args, train_data[1] + train_data[2] + dev_data[1] + dev_data[2],emb_f=args.embedding_f)
    import pdb
    #pdb.set_trace()
    train_data = vectorize(examples=train_data,word_dict=word_2_id_dict,args=args)
    dev_data = vectorize(examples=dev_data,word_dict=word_2_id_dict,args=args)
    target_train_f = os.path.join(args.source_dir,args.version,'featurized','train.json')
    import pdb
    #pdb.set_trace()
    train_data['pretrained_emb'] = pretrained_emb
    with open(target_train_f,'w') as fw:
        fw.write(json.dumps(train_data,ensure_ascii=False))
    target_dev_f = os.path.join(args.source_dir,args.version,'featurized','dev.json')
    with open(target_dev_f,'w') as fw:
        fw.write(json.dumps(dev_data,ensure_ascii=False))
    with open(os.path.join(args.source_dir, args.version,'featurized','vocab.file'),'w') as fw:
        fw.write('\n'.join(word_2_id_dict.keys()))

def get_args():
    parser = argparse.ArgumentParser()
    source_dir = '/home/yndeng/MRC/data/CrossTest'
    version = 'pure_ch'
    glove_dir = os.path.join("data", "glove")
    en_glove_path = os.path.join(glove_dir, 'en_vectors.txt')
    ch_glove_path = os.path.join(glove_dir, 'ch_vectors.txt')
    parser.add_argument('-s', "--source_dir", default=source_dir)
    parser.add_argument("--version", default=version)
    parser.add_argument('-d', "--debug", action='store_true')
    parser.add_argument("--train_ratio", default=1.0, type=int)
    parser.add_argument("--ch_glove_corpus", default=en_glove_path, type=str)
    parser.add_argument("--en_glove_corpus", default=ch_glove_path, type=str)
    parser.add_argument("--embedding_f", default='../data/dim_300/mix_vectors.txt', type=str)
    parser.add_argument("--glove_vec_size", default=300, type=int)
    parser.add_argument("--mode", default="full", type=str)
    parser.add_argument("--port", default=8000, type=int)
    parser.add_argument("--cut_q", default=False, type=bool)
    parser.add_argument("--boundary", default=10, type=int)
    parser.add_argument("--candidate_nums", default=10, type=int)
    parser.add_argument("--no_passages", default=False, type=bool)

    # TODO : put more args here
    return parser.parse_args()

def save(args, data, shared, data_type):
    data_path = os.path.join(args.target_dir, "data_{}.json".format(data_type))
    shared_path = os.path.join(args.target_dir, "shared_{}.json".format(data_type))
    json.dump(data, open(data_path, 'w'))
    json.dump(shared, open(shared_path, 'w'))

def get_word2vec(args, word_counter):
    word2vec_dict = {}
    with open(args.en_glove_corpus, 'r', encoding='utf-8') as fh:
        for line in tqdm(fh):
            array = line.lstrip().rstrip().split(" ")
            word = array[0]
            vector = list(map(float, array[1:]))
            if word in word_counter:
                word2vec_dict[word] = vector
            elif word.capitalize() in word_counter:
                word2vec_dict[word.capitalize()] = vector
            elif word.lower() in word_counter:
                word2vec_dict[word.lower()] = vector
            elif word.upper() in word_counter:
                word2vec_dict[word.upper()] = vector
    with open(args.ch_glove_corpus, 'r', encoding='utf-8') as fh:
        for line in tqdm(fh):
            array = line.lstrip().rstrip().split(" ")
            word = array[0]
            vector = list(map(float, array[1:]))
            if word in word_counter:
                word2vec_dict[word] = vector
            elif word.capitalize() in word_counter:
                word2vec_dict[word.capitalize()] = vector
            elif word.lower() in word_counter:
                word2vec_dict[word.lower()] = vector
            elif word.upper() in word_counter:
                word2vec_dict[word.upper()] = vector

    print("{}/{} of word vocab have corresponding vectors in {} and {}".format(len(word2vec_dict), len(word_counter),\
                        args.ch_glove_corpus, args.en_glove_corpus))
    return word2vec_dict

def padding_seq(seq):
    results = []
    max_len = 0
    for s in seq:
        if max_len < len(s):
            max_len = len(s)
    for i in range(0, len(seq)):
        l = max_len - len(seq[i])
        results.append(seq[i] + [0 for j in range(l)])
    return results

def get_minibatches(n, minibatch_size, shuffle=False):
    idx_list = np.arange(0, n, minibatch_size).astype(np.int32)
    if shuffle:
        np.random.shuffle(idx_list)
    minibatches = []
    import pdb
    for idx in idx_list:
        tmp = np.arange(idx, min(idx+minibatch_size, n))
        if len(tmp) < minibatch_size:
            tmp1 = list(tmp)
            tmp2 = list(np.arange(0, minibatch_size-len(tmp)))
            tmp = np.array(tmp1 + tmp2)
        minibatches.append(tmp)
    return minibatches

def build_dict(args, sentences, emb_f, max_words=50000):
    """
        Build a dictionary for the words in `sentences`.
        Only the max_words ones are kept and the remaining will be mapped to <UNK>.
        """

    word_count = Counter()
    for sent in sentences:
        for w in sent.split(' '):
            word_count[w] += 1
    max_words = int(len(word_count.keys()) * 0.95)
    ls = word_count.most_common(max_words)
    logging.info('#Words: %d -> %d' % (len(word_count), len(ls)))
    for key in ls[:5]:
        logging.info(key)
    logging.info('...')
    for key in ls[-5:]:
        logging.info(key)
    word_dict = {w[0]: index for (index, w) in enumerate(ls)}
    pre_trained_emb = []
    pre_trained_words = []
    #embeddings = np.random.randn(len(word_dict.keys()),args.glove_vec_size)
    j = 0
    for line in open(emb_f).readlines():
        sp = line.split()
        if len(sp) == 2 and int(sp[1]) == args.word_emb_size:
            continue
        assert len(sp) == args.word_emb_size + 1
        if sp[0] in word_dict and sp[0] not in pre_trained_words:
            assert word_dict[sp[0]] >= 0
            pre_trained_emb.append([float(x) for x in sp[1:]])
            pre_trained_words.append(sp[0])
            j += 1
    not_in_w2v = []
    for w in word_dict.keys():
        if w not in pre_trained_words:
            not_in_w2v.append(w)
    assert 'unk' not in pre_trained_words
    new_word_dict = ['unk'] + not_in_w2v + pre_trained_words
    assert len(new_word_dict) == len(word_dict.keys()) + 1
    return {w: index for (index, w) in enumerate(new_word_dict)}, {index: w for (index, w) in enumerate(new_word_dict)},\
           np.array(pre_trained_emb, dtype=np.float32)

def gen_examples(data, batch_size):
    """
        Divide examples into batches of size `batch_size`.
    """
    minibatches = get_minibatches(len(data['ids']), batch_size)
    import pdb
    #pdb.set_trace()
    for minibatch in minibatches:
        mb_x0 = [data['ids'][t] for t in minibatch]
        mb_x1 = [data['doc'][t] for t in minibatch]
        mb_x2 = [data['que'][t] for t in minibatch]

        mb_y = [data['answer'][t] for t in minibatch]
        mb_c = [data['candidates'][t] for t in minibatch]

        mb_x1, mb_mask1 = prepare_data(mb_x1)
        mb_x1_len = mb_mask1.sum(axis=1)
        mb_x2, mb_mask2 = prepare_data(mb_x2)
        mb_x2_len = mb_mask2.sum(axis=1)
        yield {'ids': mb_x0, 'doc': mb_x1, 'que': mb_x2,
               'doc_len': mb_x1_len, 'que_len': mb_x2_len,
               'doc_mask': mb_mask1, "que_mask": mb_mask2,
               'candidates': mb_c, 'y': mb_y}
#        all_ex.append((mb_x0, mb_x1, mb_mask1, mb_x2, mb_mask2, mb_c, mb_y))
#    return all_ex

def prepro_each(in_file, start_ratio=0, end_ratio=1):

    """
        load CNN / Daily Mail data from {train | dev | test}.txt
        relabeling: relabel the entities by their first occurence if it is True.
    """
    ids = []
    documents = []
    source_questions = []
    target_questions = []
    answers = []
    candidates = []
    f = open(in_file, 'r')
    i = 0
    lines = f.readlines()
    l = len(lines)
    from_id = int(start_ratio*l)
    to_id = int(end_ratio*l)
    for idx, line in enumerate(lines[from_id:to_id]):
        #print(idx)
        sample = json.loads(line.strip('\n'))
        ids.append(sample['id'])
        target_question = (' '.join([w[0] for w in sample['question']])).replace('\n', '')
        assert 'XXXX' in target_question
        target_questions.append(target_question)
        candidates.append([c[0] for c in sample['candidates']])
        answers.append(sample['answer'][0])
        doc = []
        for j in range(len(sample['passage'])):
            doc.append((' '.join([w[0] for w in sample['passage'][j]])).replace('\n', ''))
        doc = ' '.join(doc)
        documents.append(doc)
        i += 1
    f.close()
    logging.info('#Examples: %d' % len(documents))
    return ids, documents, target_questions, answers, candidates

def vectorize(examples, word_dict, args,
              sort_by_len=True, verbose=True):
    """
        Vectorize `examples`.
        in_x1, in_x2: sequences for document and question respecitvely.
        in_y: label
        """

    def get_word_id(w, word_dict):
        if w in word_dict:
            return word_dict[w]
        else:
            return word_dict['unk']

    in_x0 = []  # ids
    in_x1 = []  # doc
    in_x2 = []  # que
    in_y = np.zeros((len(examples[0]), 1)).astype(np.float32)
    in_c = []
    n = 0
    #import pdb
    #pdb.set_trace()
    for idx, (m_id, d, t_q, a, c) in enumerate(zip(examples[0], examples[1], examples[2], examples[3], examples[4])):
        d_words = d.split(' ')
        t_q_words = t_q.split(' ')
        if args.cut_q:
            q_pos = q_words.index('XXXX')
            q_words = q_words[q_pos - args.boundary:q_pos + args.boundary]
        if args.no_passages:
            seq1 = np.zeros(10)
        else:
            seq1 = [get_word_id(w, word_dict) for w in d_words]

        seq2 = [get_word_id(w, word_dict) for w in t_q_words]

        if a not in word_dict:
            logging.info("answer word %s not in word_dict " % a)
            continue
        if (len(seq1) > 0) and (len(seq2) > 0):
            if args.monolingual:
                if word_dict[a] not in seq1:
                    continue
            in_x0.append(m_id)
            in_x1.append(seq1)
            in_x2.append(seq2)
            tmp = []
            for j in range(args.candidate_nums):
                w_id = get_word_id(c[j], word_dict)
                tmp.append(w_id)
            random.shuffle(tmp)
            in_y[n, 0] = tmp.index(word_dict[a])
            assert word_dict[a] in tmp
            in_c.append(tmp)
            n += 1
        else:
            print(a)
    print(sum(map(len, in_x1)) / len(in_c))
    print(sum(map(len, in_x2)) / len(in_c))
    print('Vectorization: processed %d / %d' % (n, len(examples[0])))
    logging.info('Vectorization: processed %d / %d' % (n, len(examples[0])))
    in_y = in_y[:n]
    assert len(in_y) == len(in_x1)
    assert len(in_x0) == len(in_x1) == len(in_x2)

    def len_argsort(seq):
        return sorted(range(len(seq)), key=lambda x: len(seq[x]))

    if sort_by_len:
        # sort by the document length
        sorted_index = len_argsort(in_x1)
        in_x0 = [in_x0[i] for i in sorted_index]
        in_x1 = [in_x1[i] for i in sorted_index]
        in_x2 = [in_x2[i] for i in sorted_index]
        in_y = [in_y[i].tolist() for i in sorted_index]
        in_c = [in_c[i] for i in sorted_index]

    return {'ids': in_x0,
            'doc': in_x1,
            'que': in_x2,
            'candidates': in_c,
            'answer': in_y}

def prepare_data(seqs):
    lengths = [len(seq) for seq in seqs]
    n_samples = len(seqs)
    max_len = np.max(lengths)
    x = np.zeros((n_samples, max_len)).astype('int32')
    x_mask = np.zeros((n_samples, max_len), np.float32)
    for idx, seq in enumerate(seqs):
        x[idx, :lengths[idx]] = seq
        x_mask[idx, :lengths[idx]] = 1.0
    return x, x_mask

if __name__ == "__main__":
    import pdb
    #pdb.set_trace()
    main()
    load_data()


from datetime import datetime

import pytz as pytz

from bin.rss.RSSFeedGen import RSSFeedGen
import requests
from bs4 import BeautifulSoup

def UnionMangaRssHelper(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    list_cap_tags = soup.select('div > div > a[href*="/leitor"]')
    list_cap_date_tags = soup.select('div > div > span[style*="font-size: 10px; color: #999;"]')
    name = soup.find('h2')
    rss = RSSFeedGen()
    rss.set_header(str(name.next), url)
    timezone = pytz.timezone("UTC")

    for i in range(list_cap_tags.__len__()-1, -1, -1):
        cap = list_cap_tags[i]
        cap_date = list_cap_date_tags[i]
        with_timezone = timezone.localize(datetime.strptime(str(cap_date.next), '(%d/%m/%Y)'))
        rss.add_entry(str(cap.next), cap['href'], with_timezone)
    return rss.get_rss()

if __name__ == '__main__':
    UnionMangaRssHelper("https://unionmangas.top/perfil-manga/shingeki-no-kyojin")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat 19 Oct 2019 02:34:47 AM EDT

@author: 77ph
"""
### версия использовавшаяся в тестах netem

import requests
import json

auth_token="127859963:525cec3227216c14addebfc4241793c6ef4e"
url="https://api.contest.com/voip" + auth_token + "/getConnection?call={call}"
querystring=""
r=requests.get(url, params=querystring)
acc = r.json()
print(json.dumps(acc, indent=2))

if acc['ok'] == True:
        cfg = acc["result"]["config"]
        encryption_key = acc["result"]["encryption_key"]
        id = acc["result"]["endpoints"][0]["id"]
        ipv4 = acc["result"]["endpoints"][0]["ip"]
        port = acc["result"]["endpoints"][0]["port"]
        peer_tags_caller = acc["result"]["endpoints"][0]["peer_tags"]["caller"]
        peer_tags_callee = acc["result"]["endpoints"][0]["peer_tags"]["callee"]

        print("----")
        print(cfg)
        print("----")
        print(encryption_key)
        print("----")
        print(id)
        print("----")
        print(ipv4)
        print("----")
        print(port)
        print("----")
        print(peer_tags_caller)
        print("----")
        print(peer_tags_callee)
        print("----")
        hs = open("config.json", 'w')
        hs.write(json.dumps(cfg))
        hs.close()

        #./tgvoipcall 127.0.0.1:555 ssssssssss -k encryption_key_hex -i /path/to/sound_A.ogg -o /path/to/sound_output_B.ogg -c config.json
        ss = open("start_A.sh","w")
        run = "../cpp/tgvoipcall " + ipv4 + ":" + port + " " + peer_tags_caller + " -k " + encryption_key + " -i ../python/sample17.ogg -o ../python/sound_output_B.raw -c ../python/config.json" + " -l " + id + " -r caller"
        ss.write(run)
        ss.close()
        ss = open("start_B.sh","w")
        run = "../cpp/tgvoipcall " + ipv4 + ":" + port + " " + peer_tags_callee + " -k " + encryption_key + " -i ../python/sample17.ogg  -o ../python/sound_output_A.raw -c ../python/config.json" + " -l " + id + " -r callee"
        ss.write(run)
        ss.close()


# This script queries MAST for WFC3 IR data and downloads the data from
# the AWS public dataset rather than from MAST servers.

# Working with http://astroquery.readthedocs.io/en/latest/mast/mast.html
from astroquery.mast import Observations
import boto3
import os
import datetime

t_start = datetime.datetime.now()

# Read in the AWS credentials
import configparser
config = configparser.ConfigParser()
config.read(os.path.expanduser('~/.aws/credentials'))
os.environ["AWS_ACCESS_KEY_ID"] = config.get('default','aws_access_key_id')
os.environ["AWS_SECRET_ACCESS_KEY"] = config.get('default','aws_secret_access_key')

# Create a directory for downloaded files
if not os.path.exists('DATA'):
  os.makedirs('DATA')

# This downloads all the F160W DRZ images from CANDELS in the GOODS-South field
print('\nQuerying the MAST archive.\n')
obsTable = Observations.query_criteria(obs_collection='HST',
                                       filters='F160W',
                                       instrument_name='WFC3/IR',
                                       proposal_id=['12062','12061','12062'])

# Grab the list of available data products for these observations
products = Observations.get_product_list(obsTable)

# Select only drizzled (DRZ) files
filtered = Observations.filter_products(products,
                                        mrp_only=False,
                                        productSubGroupDescription='DRZ')

# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
Observations.enable_s3_hst_dataset()

# Grab the S3 URLs for each of the observations
s3_urls = Observations.get_hst_s3_uris(filtered)

print('Query returned {} entries.\n'.format(len(filtered)))

print('Downloading data from S3.\n')

s3 = boto3.resource('s3')

# Create an authenticated S3 session. Note, download within US-East is free
# e.g. to a node on EC2.
s3_client = boto3.client('s3')

bucket = s3.Bucket('stpubdata')

# For brevity, only download the first three images
for url in s3_urls[:3]:
  # Extract the S3 key from the S3 URL
  fits_s3_key = url.replace("s3://stpubdata/", "")
  root = url.split('/')[-1]
  if not os.path.exists('DATA/{}'.format(root)):
    bucket.download_file(fits_s3_key, 'DATA/{}'.format(root), ExtraArgs={"RequestPayer": "requester"})
    print('{} ---> DATA/{}'.format(fits_s3_key, root))
  else:
    print('DATA/{} already exists.'.format(root))
print('\n')

t_end = datetime.datetime.now()
duration = t_end - t_start

print('Done.\nDuration: {:6.2f} seconds.\n'.format(duration.total_seconds()))


#!/usr/bin/env python3
import datetime
import os
import queue
import re
import sys
import threading

import requests

from api import PixivApi
from model import PixivIllustModel

_THREADING_NUMBER = 5
_queue_size = 0
_finished_download = 0
_CREATE_FOLDER_LOCK = threading.Lock()
_PROGRESS_LOCK = threading.Lock()

def get_default_save_path():
    current_path = os.path.dirname(os.path.abspath(sys.argv[0]))
    return os.path.join(current_path, 'illustrations')

def get_file_path(illust, save_path='.'):
    """chose a file path by user id and name, if the current path has a folder start with the user id,
    use the old folder instead

    Return:
        file_path: a string represent complete folder path.
    """
    user_id = illust.user_id
    user_name = illust.user_name

    cur_dirs = list(filter(os.path.isfile, os.listdir(save_path)))
    cur_user_ids = [cur_dir.split()[0] for cur_dir in cur_dirs]
    if user_id not in cur_user_ids:
        dir_name = re.sub(r'[<>:"/\\|\?\*]', ' ', user_id + ' ' + user_name)
    else:
        dir_name = list(filter(lambda x: x.split()[0] == user_id, cur_dirs))[0]

    file_path = os.path.join(save_path, dir_name)

    return file_path

def print_progress():
    with _PROGRESS_LOCK:
        global _finished_download
        _finished_download += 1

        number_of_sharp = round(_finished_download / _queue_size * 60)
        number_of_space = 60 - number_of_sharp
        sys.stdout.write('\r' + str(_finished_download) + '/' + str(
            _queue_size) + '[' + '#' * number_of_sharp + ' ' * number_of_space + ']')

def download_threading(download_queue, save_path='.', add_rank=False):
    headers = {'Referer': 'http://www.pixiv.net/'}
    while not download_queue.empty():
        illust = download_queue.get()
        try:
            for url in illust.image_urls:
                file_name = url.split('/')[-1]
                if add_rank:
                    file_name = illust.rank + ' - ' + file_name
                file_path = os.path.join(save_path, file_name)
                if not os.path.exists(file_path):
                    with _CREATE_FOLDER_LOCK:
                        if not os.path.exists(os.path.dirname(file_path)):
                            os.makedirs(os.path.dirname(file_path))
                    r = requests.get(url, headers=headers, stream=True)
                    if r.status_code == requests.codes.ok:
                        temp_chunk = r.content
                        with open(file_path, 'wb') as f:
                            f.write(temp_chunk)
            print_progress()
        except Exception as e:
            print(e)
            download_queue.put(illust)
        finally:
            download_queue.task_done()

def start_and_wait_download_threadings(download_queue, save_path='.', add_rank=False):
    """start download threadings and wait till complete"""
    th = []
    for _ in range(_THREADING_NUMBER):
        t = threading.Thread(target=download_threading, args=(download_queue, save_path, add_rank))
        t.start()
        th.append(t)

    for t in th:
        t.join()

    print('\nFininsh')

def download_illusts(data_list, save_path='.', add_user_folder=False, add_rank=False):
    illusts = PixivIllustModel.from_data(data_list)

    if len(illusts) > 0:
        print('Start download, total illusts', len(illusts))

        if add_user_folder:
            save_path = get_file_path(illusts[0], save_path)

        download_queue = queue.Queue()
        for illust in illusts:
            download_queue.put(illust)

        global _queue_size, _finished_download
        _queue_size = len(illusts)
        _finished_download = 0

        start_and_wait_download_threadings(download_queue, save_path, add_rank)

    else:
        print('There is no new illust need to download')

def download_by_user_id(user):
    save_path = get_default_save_path()
    user_id = input('Input the user id:')
    data_list = user.get_user_illusts(user_id)
    download_illusts(data_list, save_path, add_user_folder=True)

def download_by_ranking(user):
    today = str(datetime.date.today())
    save_path = os.path.join(get_default_save_path(), today + ' ranking')
    data_list = user.get_ranking_illusts(per_page=100, mode='daily')
    download_illusts(data_list, save_path, add_rank=True)

def download_by_history_ranking(user):
    date = input('Input the date:(eg:2015-07-10)')
    save_path = os.path.join(get_default_save_path(), date + ' ranking')
    data_list = user.get_ranking_illusts(date=date, per_page=100, mode='daily')
    download_illusts(data_list, save_path, add_rank=True)

def main():
    print('Pixiv Downloader 2.2'.center(80, '#'))
    user = PixivApi()
    options = {
        '1': download_by_user_id,
        '2': download_by_ranking,
        '3': download_by_history_ranking
    }

    while True:
        choose = input('Which do you want to download:\n1 From user id\n2 From ranking\n3 From history ranking\n')

        if choose in [str(i) for i in range(1, 4)]:
            options[choose](user)
        else:
            print('Wrong input!')

if __name__ == '__main__':
    sys.exit(main())


# from config import Config
from keras import backend as K
from keras.layers import Input
from keras.models import Model
from app.models.frcnn_detector.model import *
from app.models.frcnn_detector.utils import *
import time
import os
import pickle
import math

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'

class Config:

    def __init__(self):
        # Print the process or not
        self.verbose = True

        # Name of base network
        self.network = 'vgg'

        # Setting for data augmentation
        self.use_horizontal_flips = False
        self.use_vertical_flips = False
        self.rot_90 = False

        # Anchor box scales
        # Note that if im_size is smaller, anchor_box_scales should be scaled
        # Original anchor_box_scales in the paper is [128, 256, 512]
        self.anchor_box_scales = [128, 256, 512]

        # Anchor box ratios
        self.anchor_box_ratios = [[1, 1],
                                  [1. / math.sqrt(2), 2. / math.sqrt(2)],
                                  [2. / math.sqrt(2), 1. / math.sqrt(2)]]

        # Size to resize the smallest side of the image
        # Original setting in paper is 600. Set to 300 in here to save training
        # time
        self.im_size = 600

        # image channel-wise mean to subtract
        self.img_channel_mean = [103.939, 116.779, 123.68]
        self.img_scaling_factor = 1.0

        # number of ROIs at once
        self.num_rois = 10

        # stride at the RPN (this depends on the network configuration)
        self.rpn_stride = 16

        self.balanced_classes = False

        # scaling the stdev
        self.std_scaling = 4.0
        self.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]

        # overlaps for RPN
        self.rpn_min_overlap = 0.3
        self.rpn_max_overlap = 0.7

        # overlaps for classifier ROIs
        self.classifier_min_overlap = 0.1
        self.classifier_max_overlap = 0.5

        # placeholder for the class mapping, automatically generated by the
        # parser
        self.class_mapping = {0: 'shoe', 1: 'bg'}

        self.model_path = None

class detector_model:
    '''
    detector model class
    '''

    def __init__(self):
        self.C = Config()
        pass

    def load_model(self, detector_weights):
        '''
    	Loads FRCNN model from detector_path
    	'''

        # Read in Config for Testing
        # with open(detector_config, 'rb') as f_in:
        #    config = pickle.load(f_in)

        # Turn off any data augmentation at test time
        self.C.model_path = detector_weights
        self.C.use_horizontal_flips = False
        self.C.use_vertical_flips = False
        self.C.rot_90 = False
        class_mapping = self.C.class_mapping
        class_mapping = {v: k for k, v in class_mapping.items()}

        # Gather Model Inputs
        num_features = 512
        input_shape_img = (None, None, 3)
        input_shape_features = (None, None, num_features)
        img_input = Input(shape=input_shape_img)
        roi_input = Input(shape=(self.C.num_rois, 4))
        feature_map_input = Input(shape=input_shape_features)

        # Define Model Layers
        shared_layers = nn_base(img_input, trainable=True)
        num_anchors = len(self.C.anchor_box_scales) * len(
            self.C.anchor_box_ratios)
        rpn_layers = rpn_layer(shared_layers, num_anchors)
        classifier = classifier_layer(feature_map_input, roi_input,
                                      self.C.num_rois,
                                      nb_classes=len(self.C.class_mapping))

        # Build Models
        model_rpn = Model(img_input, rpn_layers)
        model_classifier_only = Model([feature_map_input, roi_input],
                                      classifier)
        model_classifier = Model([feature_map_input, roi_input], classifier)

        print('Loading weights from {}'.format(self.C.model_path))
        model_rpn.load_weights(self.C.model_path, by_name=True)
        model_classifier.load_weights(self.C.model_path, by_name=True)

        model_rpn.compile(optimizer='sgd', loss='mse')
        model_classifier.compile(optimizer='sgd', loss='mse')

        self.model_rpn = model_rpn
        self.model_classifier = model_classifier

    def predict(self, img, save_path):

        # Predict
        bbox_threshold = 0.95

        final_predictions = []

        T = {}
        P = {}

        # Create copy of original image
        img_o = img
        cv2.imwrite("app/models/frcnn_detector/output/test.jpg", img_o)
        img = img_o.copy()
        X, ratio = format_img(img, self.C)

        _, fx, fy = format_img_map(img, self.C)

        # Change X (img) shape from (1, channel, height, width) to (1, height,
        # width, channel)
        X = np.transpose(X, (0, 2, 3, 1))

        # get the feature maps and output from the RPN
        [Y1, Y2, F] = self.model_rpn.predict(X)

        R = rpn_to_roi(Y1, Y2, self.C, overlap_thresh=0.8)

        # convert from (x1,y1,x2,y2) to (x,y,w,h)
        R[:, 2] -= R[:, 0]
        R[:, 3] -= R[:, 1]

        # apply the spatial pyramid pooling to the proposed regions
        bboxes = {}
        probs = {}

        for jk in range(R.shape[0] // self.C.num_rois + 1):
            ROIs = np.expand_dims(
                R[self.C.num_rois * jk:self.C.num_rois * (jk + 1), :], axis=0)
            if ROIs.shape[1] == 0:
                break

            if jk == R.shape[0] // self.C.num_rois:
                # pad R
                curr_shape = ROIs.shape
                target_shape = (curr_shape[0], self.C.num_rois, curr_shape[2])
                ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype)
                ROIs_padded[:, :curr_shape[1], :] = ROIs
                ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
                ROIs = ROIs_padded

            [P_cls, P_regr] = self.model_classifier.predict([F, ROIs])

            # Calculate all classes' bboxes coordinates on resized image
            # (300, 400) Drop 'bg' classes bboxes
            for ii in range(P_cls.shape[1]):

                # If class name is 'bg', continue
                if np.max(P_cls[0, ii, :]) < bbox_threshold or np.argmax(
                        P_cls[0, ii, :]) == (P_cls.shape[2] - 1):
                    continue

                # Get class name
                cls_name = self.C.class_mapping[np.argmax(P_cls[0, ii, :])]

                if cls_name not in bboxes:
                    bboxes[cls_name] = []
                    probs[cls_name] = []

                (x, y, w, h) = ROIs[0, ii, :]

                cls_num = np.argmax(P_cls[0, ii, :])
                try:
                    (tx, ty, tw, th) = P_regr[0, ii,
                                       4 * cls_num:4 * (cls_num + 1)]
                    tx /= self.C.classifier_regr_std[0]
                    ty /= self.C.classifier_regr_std[1]
                    tw /= self.C.classifier_regr_std[2]
                    th /= self.C.classifier_regr_std[3]
                    x, y, w, h = roi_helpers.apply_regr(x, y, w, h, tx, ty, tw,
                                                        th)
                except:
                    pass
                bboxes[cls_name].append(
                    [16 * x, 16 * y, 16 * (x + w), 16 * (y + h)])
                probs[cls_name].append(np.max(P_cls[0, ii, :]))

        all_dets = []
        real_dets = []

        # if no predictions
        if bboxes == {}:
            real_dets.append(
                {'x1': 0, 'y1': 0, 'x2': img.shape[0], 'y2': img.shape[1]})

        for key in bboxes:
            bbox = np.array(bboxes[key])

            # Apply non-max-suppression on final bboxes to get the output
            # bounding boxe
            new_boxes, new_probs = non_max_suppression_fast(
                bbox, np.array(probs[key]), overlap_thresh=0.2)
            for jk in range(new_boxes.shape[0]):
                (x1, y1, x2, y2) = new_boxes[jk, :]
                det = {'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2,
                       'class': key, 'prob': new_probs[jk]}
                all_dets.append(det)

                # Calculate real coordinates on original image
                (real_x1, real_y1, real_x2, real_y2) = get_real_coordinates(
                    ratio, x1, y1, x2, y2)
                real_det = {'x1': real_x1, 'x2': real_x2, 'y1': real_y1,
                            'y2': real_y2, 'prob': new_probs[jk]}
                real_dets.append(real_det)

        final_predictions.append([img_o, real_dets])
        print(len(final_predictions[0][1]))

        # Crop images and save
        img_paths = save_crop(final_predictions, save_path)

        return img_paths


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/betsee/gui/simconf/stack/page/export/guisimconfpageexport.py
# Compiled at: 2019-08-01 01:03:54
# Size of source mod 2**32: 7385 bytes
"""
**Export simulation configuration pager** (i.e., :mod:`PySide2`-based
controller for stack widget pages specific to export settings) functionality.
"""
from PySide2.QtWidgets import QMainWindow
from betse.science.parameters import Parameters
from betse.science.config.export.visual.confexpvisual import SimConfExportVisual
from betse.lib.matplotlib import mplcolormap
from betse.util.type.types import type_check
from betsee.util.widget.abc.control.guictlpageabc import QBetseePagerABC

class QBetseeSimConfPagerExport(QBetseePagerABC):
    __doc__ = '\n    **Export simulation configuration pager** (i.e., :mod:`PySide2`-based\n    controller connecting all editable widgets of the export stack widget page\n    with corresponding settings of the current simulation configuration).\n    '

    @type_check
    def init(self, main_window):
        super().init(main_window)
        sim_conf_visual = main_window.sim_conf.p.visual
        main_window.sim_conf_exp_is_show_cell_indices.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(SimConfExportVisual.is_show_cell_indices),
          sim_conf_alias_parent=sim_conf_visual)
        main_window.sim_conf_exp_single_cell_index.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(SimConfExportVisual.single_cell_index),
          sim_conf_alias_parent=sim_conf_visual)
        colormap_names = mplcolormap.iter_colormap_names()
        main_window.sim_conf_exp_colormap_diverging.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(Parameters.colormap_diverging_name),
          items_iconless_text=colormap_names)
        main_window.sim_conf_exp_colormap_sequential.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(Parameters.colormap_sequential_name),
          items_iconless_text=colormap_names)
        main_window.sim_conf_exp_colormap_gj.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(Parameters.colormap_gj_name),
          items_iconless_text=colormap_names)
        main_window.sim_conf_exp_colormap_grn.init(sim_conf=(main_window.sim_conf),
          sim_conf_alias=(Parameters.colormap_grn_name),
          items_iconless_text=colormap_names)


import operator
from math import log

import matplotlib.pyplot as plt

# 决策树算法流程 （ID3）
# 1 准备数据
# 2 按照给定的特征划分数据集 如：属性A =1 时 剩余数据组成的矩阵
# 3 求香农熵函数 把数据集按照最后结果分类成dict类型的数据 如：{'yes': 2, 'no': 3}
#     公式                 c
#          Entropy(S) =   Σ - pi log2pi
#                         i=1
# 4 求最大增益属性  循环各个属性求出增益值  取出最大的一个
# 5 使用dict 构建出决策树完整模型

# 使用matplotlib 描绘决策树
# 1 定义总体样式(x轴 y轴 backgroud颜色  箭头....)  note： 乱码处理
# 2 决策树叶子节点数目
# 3 决策树层数
# 4 节点文本框样式
# 5 描绘树的形状

# 求给定数据集的香农熵  note:当dataset中有一种类型的数据时 熵为0  当存在两种类型的数据 并且数量相同时 熵为1
def calcShannonEnt(dataSet):
    numEntries = len(dataSet)
    labelCounts = {}  # {'yes': 2, 'no': 3}
    # 为所有分类创建字典 start
    for featVec in dataSet:
        currentlabel = featVec[-1]
        if currentlabel not in labelCounts.keys():
            # 如果分类不在labelCounts 中 就添加进去
            labelCounts[currentlabel] = 0
        labelCounts[currentlabel] += 1
    # 为所有分类创建字典 end
    shannonEnt = 0.0
    for key in labelCounts:
        # labelCounts中各分类占总分类的比例
        prob = float(labelCounts[key] / numEntries)
        # 求以2为底的对数
        #                c
        # Entropy(S) =   Σ -pi log2pi
        #               i=1
        shannonEnt -= prob * log(prob, 2)
    return shannonEnt

# 数据
def createDataSet():
    dataSet = [
        [1, 1, 'yes'],
        [1, 1, 'yes'],
        [1, 0, 'no'],
        [0, 1, 'no'],
        [1, 1, 'no']
    ]
    labels = ['no surfacing', 'flippers']
    return dataSet, labels

# 按照给定的特征划分数据集:找到对应角标的属性值==value数据 并且删掉该角标对应的属性值 最后输出
# 参数            数据集  属性对应的角标 属性值
# [[1, 'yes'], [1, 'yes'], [0, 'no'], [1, 'no']]
def splitDataSet(dataset, axis, value):
    retDataSet = []
    for featVec in dataset:
        if featVec[axis] == value:
            reduceFeatVec = featVec[:axis]
            reduceFeatVec.extend(featVec[axis + 1:])
            retDataSet.append(reduceFeatVec)
    return retDataSet

# 求最大增益属性对应的角标
def chooseMaxBenefitProperty(dataset):
    numFeatures = len(dataset[0]) - 1
    # 香农熵
    baseEntroy = calcShannonEnt(dataset)
    bestInfoGain = 0.0;
    bestFeature = -1
    for i in range(numFeatures):
        # 获取训练集中的某一属性的所有值比如dataset中第一个属性 [1,1,1,0,1]组成元组
        featList = [example[i] for example in dataset]
        # 去重复 [0,1]
        uniqueVals = set(featList)
        newEntroy = 0.0
        for value in uniqueVals:
            # 按照给定的特征划分数据集
            subDataSet = splitDataSet(dataset, i, value)
            # 求该特征集占总量的比例
            prob = len(subDataSet) / float(len(dataset))
            newEntroy += prob * calcShannonEnt(subDataSet)
        # 该属性的增益
        infoGain = baseEntroy - newEntroy
        # 最大增益属性
        if (infoGain > bestInfoGain):
            bestInfoGain = infoGain
            # 最大增益属性对应的角标
            bestFeature = i
    return bestFeature

# 当数据集已经处理了所有属性，但是类标签已然不是唯一的，如何定义该叶子节点
def majorityCnt(classList):
    classCount = {}
    for vote in classList:
        if vote not in classCount.keys(): classCount[vote] = 0
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

# 使用dict类型来模拟决策树结构
def createTree(dataSet, labels):
    classList = [example[-1] for example in dataSet]
    # 类别相同停止划分
    if classList.count(classList[0]) == len(classList):
        return classList[0]
    # 遍历完所有特征时返回出现次数最多的类别
    if len(dataSet[0]) == 1:
        return majorityCnt(classList)
    # 最大增益
    bestFeat = chooseMaxBenefitProperty(dataSet)
    bestFeatLabel = labels[bestFeat]
    myTree = {bestFeatLabel: {}}
    del (labels[bestFeat])
    featValues = [example[bestFeat] for example in dataSet]
    uniqueVals = set(featValues)
    for value in uniqueVals:
        subLabels = labels[:]
        myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value), subLabels)
    return myTree

# 决策节点文本框格式
decisionNode = dict(boxstyle="sawtooth", fc="0.8")
# 叶子节点文本框格式
leafNode = dict(boxstyle="round4", fc="0.8")
# 箭头格式
arrow_args = dict(arrowstyle="<-")
# matplotlib中文乱码
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号

# matplotlib绘图测试
def createPlotTest():
    # x和y轴长度   颜色
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    createPlotTest.axl = plt.subplot(111, frameon=False)
    # 节点名称  节点坐标
    plotNode('决策节点', (0.5, 0.1), (0.1, 0.5), decisionNode)
    plotNode('叶子节点', (0.8, 0.1), (0.3, 0.8), leafNode)
    plt.show()

def plotNode(nodeTxt, centerPt, parentPt, nodeType):
    createPlot.axl.annotate(nodeTxt, xy=parentPt, xycoords='axes fraction', xytext=centerPt, textcoords='axes fraction',
                            va="center", ha="center", bbox=nodeType, arrowprops=arrow_args)

# 获取叶子节点的数目 (叶子节点:已经获得最终分类结果 )
def getNumLeaf(myTree):
    numLeaf = 0
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        # 如果该节点下还有数据 -> 递归
        if type(secondDict[key]).__name__ == 'dict':
            numLeaf += getNumLeaf(secondDict[key])
        else:
            numLeaf += 1
    return numLeaf

# 获取树的层数
def getTreeDepth(myTree):
    maxDepth = 0
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            thisDepth = 1 + getTreeDepth(secondDict[key])
        else:
            thisDepth = 1
        # 取最高的层数
        if thisDepth > maxDepth:
            maxDepth = thisDepth
    return maxDepth

# 决策树绘图数据准备
def retrieveTree(i):
    listOfTrees = [{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
                   {'no surfacing': {0: 'no', 1: {'flippers': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}]
    return listOfTrees[i]

# 父子节点填充文本信息
def plotMidText(cntrpt, parentPt, txtString):
    xMid = (parentPt[0] - cntrpt[0]) / 2.0 + cntrpt[0]
    yMid = (parentPt[1] - cntrpt[1]) / 2.0 + cntrpt[1]
    createPlot.axl.text(xMid, yMid, txtString)

def plotTree(myTree, parentPt, nodeTxt):
    numLeafs = getNumLeaf(myTree)
    depth = getTreeDepth(myTree)
    firstStr = list(myTree.keys())[0]
    cntrPt = (plotTree.xOff + (1.0 + float(numLeafs)) / 2.0 / plotTree.totalW, plotTree.yOff)
    plotMidText(cntrPt, parentPt, nodeTxt)
    plotNode(firstStr, cntrPt, parentPt, decisionNode)
    secondDict = myTree[firstStr]
    plotTree.yOff = plotTree.yOff - 1.0 / plotTree.totalD
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            plotTree(secondDict[key], cntrPt, str(key))
        else:
            plotTree.xOff = plotTree.xOff + 1.0 / plotTree.totalW
            plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode)
            plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
    plotTree.yOff = plotTree.yOff + 1.0 / plotTree.totalD

# 创建决策树
def createPlot(intree):
    # create figure
    flg = plt.figure(1, facecolor='white')
    flg.clf()
    axprops = dict(xticks=[], yticks=[])
    # 创建坐标轴
    createPlot.axl = plt.subplot(111, frameon=False, **axprops)
    plotTree.totalW = float(getNumLeaf(intree))
    plotTree.totalD = float(getTreeDepth(intree))
    plotTree.xOff = -0.5 / plotTree.totalW;
    plotTree.yOff = 1.0
    plotTree(intree, (0.5, 1.0), '')
    plt.show()

# 使用决策树对数据进行分类
def classify(inputTree, featLabels, testVec):
    firstStr = list(inputTree.keys())[0]
    secondDict = inputTree[firstStr]
    # 这里假设测试数据中的属性排列顺序和 label 中一致
    featIndex = featLabels.index(firstStr)
    for key in secondDict:
        if testVec[featIndex] == key:
            if type(secondDict[key]).__name__ == 'dict':
                classLabel = classify(secondDict[key], featLabels, testVec)
            else:
                classLabel = secondDict[key]
    return classLabel

# 求香农熵
def myShannonEnt(dataSet):
    dataCount = {}  # {'yes': 2, 'no': 3}
    for data in dataSet:
        key = data[-1]
        dataCount[key] = dataCount.get(key, 0) + 1
    # 求香农熵
    shannonEnt = 0.0
    totalEntries = len(dataSet)
    # 字典类型迭代
    for key in dataCount:
        shannonEnt -= dataCount.get(key) / totalEntries * log(dataCount.get(key) / totalEntries, 2)
    return shannonEnt

# 按照属性划分特征集
# 找出第index个且值为value的特征集
def splitDataSetByProp(dataSet, index, value):
    character = []
    for array in dataSet:
        if array[index] == value:
            character.append([array[index], array[-1]])
    return character

# 求最大增益
def maxBenefitValue(dataSet, label):
    maxBenefitVal = 0.0
    maxBenefitPropIndex = None
    shannonEnt = myShannonEnt(dataSet)
    # 训练样本中属性个数
    propertyNum = len(dataSet[0]) - 1
    # 遍历所有属性
    for i in range(propertyNum):
        # 当前属性所有值
        propList = [array[i] for array in dataSet]
        propSet = set(propList)
        minusShannonEnt = 0.0
        for prop in propSet:
            character = splitDataSetByProp(dataSet, i, prop)
            # 该属性所有值对应的熵值相加
            minusShannonEnt += myShannonEnt(character) * len(character) / float(len(dataSet))
        # 该属性增益和最大比较
        if shannonEnt - minusShannonEnt > maxBenefitVal:
            maxBenefitVal = shannonEnt - minusShannonEnt
            maxBenefitPropIndex = i
    return maxBenefitPropIndex

# 使用dict描述树的形状
def createTreeModelByDict(dataSet, label):
    maxBenefitPropIndex = maxBenefitValue(dataSet, label)
    benefitPropList = [array[maxBenefitPropIndex] for array in dataSet]
    benefitPropSet = set(benefitPropList)

if __name__ == '__main__':
    myDat, labels = createDataSet()
    # 求数据集熵值
    # print(calcShannonEnt(myDat))
    # 属性分类
    # print(splitDataSet(myDat,0,1))
    # 最大增益属性
    # print(chooseMaxBenefitProperty(myDat))
    # 创建tree的dict结构
    # print(createTree(myDat, labels))
    # matplotlib绘图测试
    # myTree = retrieveTree(0)
    # myTree1 = retrieveTree(1)
    # print(getTreeDepth(myTree))
    # createPlot(myTree1)
    # print(matplotlib.matplotlib_fname())  #获取matplotlib包所在路径
    # print(classify(myTree,labels,[1,0]))
    # print(myShannonEnt(myDat))
    print(maxBenefitValue(myDat, labels))
# [
#         [1, 1, 'yes'],
#         [1, 1, 'yes'],
#         [1, 0, 'no'],
#         [0, 1, 'no'],
#         [1, 1, 'no']
#     ]


from setuptools import setup, find_packages

version = '1.0.0'

setup(
    name='ODP-Identity',
    version=version,
    description='The SAEON Open Data Platform Identity Service',
    url='https://github.com/SAEONData/ODP-Identity',
    author='Mark Jacobson',
    author_email='mark@saeon.ac.za',
    license='MIT',
    packages=find_packages(),
    include_package_data=True,
    python_requires='~=3.8',
    install_requires=[
        'flask',
        'flask-login',
        'flask-wtf',
        'flask-mail',
        'itsdangerous',
        'python-dotenv',
        'wtforms',
        'argon2_cffi',
    ],
    extras_require={
        'test': ['pytest', 'pytest-cov']
    },
    dependency_links=[
        'git+https://github.com/SAEONData/ODP-AccountsLib.git#egg=ODP_AccountsLib',
        'git+https://github.com/SAEONData/Hydra-Admin-Client.git#egg=Hydra_Admin_Client',
        'git+https://github.com/SAEONData/Hydra-OAuth2-Blueprint.git#egg=Hydra_OAuth2_Blueprint',
    ],
)


from flask import Flask
from flask import flash, redirect, render_template, request, url_for
from flask_sqlalchemy import SQLAlchemy
from flask_login.utils import current_user, login_user, logout_user

from config import app, db
from tables import Shuttles_A, Shuttles_B, Booking, Users
from functions import add_form, get_index_data, is_admin, debug_print, \
  is_safe_url, get_redirect_target, login_required

@app.route('/')
def index():
    shuttlesA, shuttlesB = get_index_data(db)
    return render_template('index.html', shuttlesA=shuttlesA, shuttlesB=shuttlesB)

@app.route('/login', methods = ['GET', 'POST'])
def login():
    if current_user.is_authenticated:
        return redirect(url_for('admin.index'))
    if request.method == 'GET':
        return render_template('login.html')
    user, verify = is_admin(app, db, request.form)
    if not verify:
        flash('Echec de l\'authentification.', 'error')
        return redirect(url_for('login'))
    login_user(user)
    flash('Vous êtes connecté en tant qu\'administrateur.', 'success')
    return redirect(get_redirect_target() or url_for('admin.index'))

@app.route('/logout')
@login_required
def logout():
    logout_user()
    flash('Déconnexion effectuée avec succès.', 'success')
    return redirect(url_for('index'))

@app.route('/add', methods = ['POST'])
def new():
    #fields = ['surname', 'firstname', 'presale']
    fields = Booking.__table__.columns.keys()[1:-2]
    if False in [ request.form[field] != '' for field in fields ]:
        flash('Il manque des informations pour réserver les navettes !', 'error')
        return redirect(url_for('index'))

    debug_print(app, request.form)
    add_form(app, db, request.form)

    shuttlesA, shuttlesB = Shuttles_A.query.all(), Shuttles_B.query.all()
    return redirect(url_for('index'))

@app.errorhandler(403)
def forbidden(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(404)
def not_found(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(405)
def method_not_allowed(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(500)
def internal_server_error(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

if __name__=='__main__':
    app.run(host='0.0.0.0', debug=True)