import numpy as np
import multiprocessing as mp
from itertools import repeat
import time
import pandas as pd

# Custom functions from other files we wrote
import PSOTestFuncs as tf
from PSOInit import pso_init
from PSOInit import qpso_init
from PSOUpdate import veloc_update
from PSOUpdate import point_update
from PSOUpdate import qpoint_update

############################ Contents ############################

# Defines the 4 algorithms that will be used:
#	- PSO - pso_algo()
#	- QPSO - qpso_algo()
#	- Parallelized PSO - pso_algo_par()
#	- Parallelized QPSO - qpso_algo_par()

# Runs 50 simulations of each algorithm on each test function within PSOTestFuncs.py
# Saves the output to a CSV

############################ Variable descriptions ############################

# n is the number of dimensions (int)
# s is the number of particles (int)
# bounds of the search area - of the form [[x1min, x1max], ... , [xnmin, xnmax]]
# f is the function to be optimized
# params are the necessary parameters (omega, c1, c2) for PSO
# t is the current iteration of the algorithm
# sims is number of simulations to run on each function

# maxrounds is the maximum number of iterations allowed
# tol is the amount of change in fgbest to be considered an improvement
# nochange is the number or iterations without a sufficient improvement in fgbest before stopping
# samebest is a counter for how many rounds in a row with improvement in fgbest of less than tol

# pcurr is the current position of each particles
# vcurr is the current velocity of each particles (PSO and parallelized PSO only)
# pbest is the best position of each particle
# fbest is the minimum value found for each particle
# fgbest is the overall minimum value found of all particles
# pgbest is the overall best position of each particle
# x
# mbest

# newp is a temporary calculation of the new position, saved to pcurr if inside the bounds
# newx_id is a temporary calculation of the new x, saved to x if inside the bounds
# fgbest_compare

# rpg
# phi
# u
# beta
# changeParam
# coinToss - 50% chance of being True / False, used in QPSO to decide to add or subtract changeParam

############################ Algorithm Functions ############################

# Takes in f, s, bounds, params, maxrounds, tol, nochange
# Runs PSO on f over the search area bounds using s particles and parameters params,
#	and stopping criteria specified by maxrounds, tol, nochange
# Returns pgbest, fgbest, and t

def pso_algo(f, s, bounds, params, maxrounds, tol, nochange):
    n = len(bounds)
    pcurr, vcurr, pbest, fbest, pgbest, fgbest = pso_init(f, s, bounds)
    t = 0
    samebest = 0
    while t < maxrounds:
        fgbest_compare = fgbest
        for i in range(s):
            for d in range(n):
                vcurr[i][d] = veloc_update(pcurr[i][d], vcurr[i][d], pbest[i][d], pgbest[d], params)
            newp = pcurr[i] + vcurr[i]
            for d in range(n):
                if newp[d] > bounds[d][0] and newp[d] < bounds[d][1]:
                    #Adding 0 creates a new object in memory instead of variable that references same object
                    pcurr[i][d] = newp[d] + 0
            fcurr = f(pcurr[i])
            if fcurr < fbest[i]:
                fbest[i] = fcurr + 0
                pbest[i] = pcurr[i] + 0
                if fcurr < fgbest:
                    fgbest = fcurr + 0
                    pgbest = pcurr[i] + 0
        t += 1

        if abs(fgbest_compare - fgbest) > tol :
            samebest = 0
        else :
            samebest += 1

        if samebest >= nochange :
            break
    return pgbest, fgbest, t

# Takes in f, s, bounds, maxrounds, tol, nochange
# Runs QPSO on f over the search area bounds using s particles,
#	and stopping criteria specified by maxrounds, tol, nochange
# Returns pgbest, fgbest, and t

def qpso_algo(f, s, bounds, maxrounds, tol, nochange):
    n = len(bounds)
    pcurr, pbest, fbest, pgbest, fgbest = qpso_init(f, s, bounds)
    x = np.copy(pcurr, order="k")
    t = 0
    samebest = 0
    while t < maxrounds:
        fgbest_compare = fgbest
        mbest = np.mean(pbest, axis=0)
        beta = 0.5*(maxrounds-t)/maxrounds + 0.5

        for i in range(s):
            for d in range(n):
                phi = np.random.uniform()
                u = np. random.uniform()
                coinToss = np.random.uniform() < 0.5
                pcurr[i,d] = phi*pbest[i,d] + (1- phi)*pgbest[d]
                changeParam = beta * abs(mbest[d] - x[i, d]) * (-1) * np.log(u)
                newx_id = pcurr[i, d] + changeParam if coinToss else pcurr[i, d] - changeParam
                if newx_id > bounds[d][0] and newx_id < bounds[d][1]:
                    #Adding 0 creates a new object in memory instead of variable that references same object
                    x[i,d] = newx_id + 0
            fcurr = f(x[i])
            if fcurr < fbest[i]:
                fbest[i] = fcurr + 0
                pbest[i] = x[i] + 0
                if fcurr < fgbest:
                    fgbest = fcurr + 0
                    pgbest = x[i] + 0
        t += 1

        if abs(fgbest_compare - fgbest) > tol:
            samebest = 0
        else:
            samebest += 1

        if samebest >= nochange:
            break
    return pgbest, fgbest, t

# Takes in f, s, bounds, params, maxrounds, tol, nochange
# Runs parallelized PSO on f over the search area bounds using s particles and parameters params,
#	and stopping criteria specified by maxrounds, tol, nochange
# We update all the points in an iteration at once, so no communication within an iteration
# Returns pgbest, fgbest, and t

def pso_algo_par(f, s, bounds, params, maxrounds, tol, nochange):
    n = len(bounds)
    pcurr, vcurr, pbest, fbest, pgbest, fgbest = pso_init(f, s, bounds)
    t = 0
    samebest = 0
    while t < maxrounds:
        fgbest_compare = fgbest
        inputs = zip(pcurr, vcurr, pbest, fbest, repeat(pgbest), repeat(params), repeat(bounds), repeat(f))

        results_0 = pool.starmap(point_update, inputs)
        results = list(map(list, zip(*results_0)))

        pcurr = np.array(list(results)[0])
        vcurr = np.array(list(results)[1])
        pbest = np.array(list(results)[2])
        fbest = np.array(list(results)[3])

        if min(fbest) < fgbest:
            #Adding 0 creates a new object in memory instead of variable that references same object
            fgbest = min(fbest) + 0
            pgbest = np.copy(pbest[fbest == fgbest], order="k")[0]
        t += 1

        if abs(fgbest_compare - fgbest) > tol:
            samebest = 0
        else:
            samebest += 1

        if samebest >= nochange:
            break
    return pgbest, fgbest, t

# Takes in f, s, bounds, maxrounds, tol, nochange
# Runs parallelized QPSO on f over the search area bounds using s particles and parameters params,
#	and stopping criteria specified by maxrounds, tol, nochange
# We update all the points in an iteration at once, so no communication within an iteration
# Returns pgbest, fgbest, and t

def qpso_algo_par(f, s, bounds, maxrounds, tol, nochange):
    pcurr, pbest, fbest, pgbest, fgbest = qpso_init(f, s, bounds)
    x = np.copy(pcurr, order="k")
    t = 0
    samebest = 0
    while t < maxrounds:
        fgbest_compare = fgbest
        mbest = np.mean(pbest, axis=0)
        beta = 0.5*(maxrounds-t)/maxrounds + 0.5

        inputs = zip(x, pcurr, pbest, fbest, repeat(mbest), repeat(pgbest), repeat(beta), repeat(bounds), repeat(f))

        results_0 = pool.starmap(qpoint_update, inputs)
        results = list(map(list, zip(*results_0)))

        x = np.array(list(results)[0])
        pcurr = np.array(list(results)[1])
        pbest = np.array(list(results)[2])
        fbest = np.array(list(results)[3])

        if min(fbest) < fgbest:
            #Adding 0 creates a new object in memory instead of variable that references same object
            fgbest = min(fbest) + 0
            pgbest = np.copy(pbest[fbest == fgbest], order="k")[0]

        t += 1

        if abs(fgbest_compare - fgbest) > tol:
            samebest = 0
        else:
            samebest += 1
        if samebest >= nochange:
            break
    return pgbest, fgbest, t

############################ Simulations and Testing ############################

if __name__ == '__main__':

    # Specifies the necessary parameters to be used by the algorithms, and # of simulations

    s = 50
    params = [0.715, 1.7, 1.7]
    maxrounds = 1000
    tol = 10**(-9)
    nochange = 20
    sims = 50

    # Stores the information for each function including names of function as a string,
    # how to call it, where the true minimum occurs, and what the bounds are

    funcnamelist = ["X-Squared", "Booth", "Beale", "ThreeHumpCamel", "GoldsteinPrice", "Levi_n13", "Sphere", "Rosebrock", "StyblinskiTang", "Ackley", "Schaffer_n2", "Eggholder", "McCormick", "Rastrigin", "Schaffer_n4", "Easom", "Bukin_n6", "Matyas"]
    functionlist = [tf.xsq, tf.booth, tf.beale, tf.threehumpcamel, tf.goldsteinprice, tf.levi_n13, tf.sphere, tf.rosenbrock, tf.Styblinski_Tang, tf.ackley, tf.schaffer_n2, tf.eggholder, tf.mccormick, tf.rastrigin, tf.schaffer_n4, tf.easom, tf.bukin_n6, tf.matyas]
    pminlist = [[0], [1,3], [3,0.5], [0,0], [0, -1],[1,1], [0,0,0,0], [1,1,1,1], [-2.903534,-2.903534,-2.903534,-2.903534,-2.903534,-2.903534], [0,0], [0,0], [512, 404.2319], [-0.54719, -1.54719], [0,0,0,0,0,0,0,0], [0,1.25313], [np.pi, np.pi], [-10,1], [0,0]]
    boundlist = [[[-200, 200]], [[-10, 10], [-10, 10]], [[-4.5, 4.5], [-4.5, 4.5]], [[-5, 5], [-5, 5]], [[-2, 2], [-2, 2]], [[-10, 10], [-10, 10]], [[-100, 100], [-100, 100], [-100, 100], [-100, 100]], [[-100, 100], [-100, 100], [-100, 100], [-100, 100]], [[-5, 5], [-5, 5], [-5, 5], [-5, 5], [-5, 5], [-5, 5]], [[-5, 5], [-5, 5]], [[-100, 100], [-100, 100]], [[-512, 512], [-512, 512]], [[-1.5, 4], [-3, 4]], [[-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12], [-5.12, 5.12]], [[-100, 100], [-100, 100]], [[-100, 100], [-100, 100]], [[-15, -5], [-3, 3]], [[-10.00, 10.00], [-10.00, 9.00]]]

    # Sets up a dataframe to store the data
    outdata = pd.DataFrame()

    # Sets up for parallel computing
    cores = mp.cpu_count()
    pool = mp.Pool(processes=cores)

    # Forloop for each function containing for-loop for each simulation which runs all 4 algorithms and times them
    # Stores the results of each simulation and true function values in outdata, and saves this as a CSV

    for j in range(len(functionlist)):
        for k in range(sims):
            f = functionlist[j]
            bounds = boundlist[j]
            trueval = f(pminlist[j])

            start = time.time()
            pmin, fmin, nrounds = pso_algo(f, s, bounds, params, maxrounds, tol, nochange)
            end = time.time()
            outdata = outdata.append([[k, funcnamelist[j], "PSO", end-start, nrounds, pmin, pminlist[j], fmin, trueval]])

            start = time.time()
            pmin, fmin, nrounds = pso_algo_par(f, s, bounds, params, maxrounds, tol, nochange)
            end = time.time()
            outdata = outdata.append([[k, funcnamelist[j], "PSO_Par", end-start, nrounds, pmin, pminlist[j], fmin, trueval]])

            start = time.time()
            pmin, fmin, nrounds = qpso_algo(f, s, bounds, maxrounds, tol, nochange)
            end = time.time()
            outdata = outdata.append([[k, funcnamelist[j], "QPSO", end-start, nrounds, pmin, pminlist[j], fmin, trueval]])

            start = time.time()
            pmin, fmin, nrounds = qpso_algo_par(f, s, bounds, maxrounds, tol, nochange)
            end = time.time()
            outdata = outdata.append([[k, funcnamelist[j], "QPSO_Par", end-start, nrounds, pmin, pminlist[j], fmin, trueval]])

    pool.close()
    outdata.columns = ["Simulation#", "Function", "Method", "time", "rounds", "FoundMinLoc", "TrueMinLoc", "FoundMinVal", "TrueMinVal"]
    outdata.sort_values(["Function", "Method"], inplace = True)
    outdata = outdata.reset_index(drop = True)
    outdata.to_csv("OutputData.csv")


from typing import Any, Dict
import argparse

import torch
import torch.nn as nn
import torch.nn.functional as F

CONV_DIM = 64
FC_DIM = 128
IMAGE_SIZE = 28

class ConvBlock(nn.Module):
    """
    Simple 3x3 conv with padding size 1 (to leave the input size unchanged), followed by a ReLU.
    """

    def __init__(self, input_channels: int, output_channels: int) -> None:
        super().__init__()
        self.conv = nn.Conv2d(input_channels, output_channels, kernel_size=3, stride=1, padding=1)
        self.relu = nn.ReLU()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Parameters
        ----------
        x
            of dimensions (B, C, H, W)

        Returns
        -------
        torch.Tensor
            of dimensions (B, C, H, W)
        """
        c = self.conv(x)
        r = self.relu(c)
        return r

# kernel size 3 + padding of 1 maintains the dimensionality of the input to be eq to output so we can go for skip conns here
class ResBlock(nn.Module):
    def __init__(self, input_channels: int, output_channels: int, batch_norm=False) -> None:
        super().__init__()
        self.conv1 = nn.Conv2d(input_channels, output_channels, kernel_size=3, stride=1, padding=1)
        self.bn1 = nn.Identity()
        self.conv2 = nn.Conv2d(output_channels, output_channels, kernel_size=3, stride=1, padding=1)
        self.bn2 = nn.Identity()
        self.conv_skip = nn.Conv2d(input_channels, output_channels, kernel_size=3, stride=1, padding=1)
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        if batch_norm:
            self.bn1 = nn.BatchNorm2d(output_channels)
            self.bn2 = nn.BatchNorm2d(output_channels)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Parameters
        ----------
        x
            of dimensions (B, C, H, W)

        Returns
        -------
        torch.Tensor
            of dimensions (B, C, H, W)
        """
        y = self.conv1(x)
        y = self.bn1(y)
        y = self.relu1(y)
        y = self.conv2(y)
        y = self.bn2(y)
        skip_y = self.conv_skip(x)
        y = y + skip_y
        y = self.relu2(y)
        return y

def _block(t: str, d1: int, d2: int) -> nn.Module:
    if t == 'cnn':
        return ConvBlock(d1, d2)
    if t == 'res':
        return ResBlock(d1, d2)
    if t == 'res_bn':
        return ResBlock(d1, d2, batch_norm = True)

    raise ValueError()

class CNN(nn.Module):
    """Simple CNN for recognizing characters in a square image."""

    def __init__(self, data_config: Dict[str, Any], args: argparse.Namespace = None,
        block_type = 'cnn', batch_norm = False) -> None:
        super().__init__()
        self.args = vars(args) if args is not None else {}

        input_dims = data_config["input_dims"]
        num_classes = len(data_config["mapping"])

        conv_dim = self.args.get("conv_dim", CONV_DIM)
        fc_dim = self.args.get("fc_dim", FC_DIM)

        self.conv1 = _block(block_type, input_dims[0], conv_dim)
        self.conv2 = _block(block_type, conv_dim, conv_dim)
        self.dropout = nn.Dropout(0.25)
        self.max_pool = nn.MaxPool2d(2)

        # Because our 3x3 convs have padding size 1, they leave the input size unchanged.
        # The 2x2 max-pool divides the input size by 2. Flattening squares it.
        conv_output_size = IMAGE_SIZE // 2
        fc_input_dim = int(conv_output_size * conv_output_size * conv_dim)
        self.fc1 = nn.Linear(fc_input_dim, fc_dim)
        self.fc2 = nn.Linear(fc_dim, num_classes)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Args:
        x
            (B, C, H, W) tensor, where H and W must equal IMAGE_SIZE

        Returns
        -------
        torch.Tensor
            (B, C) tensor
        """
        _B, _C, H, W = x.shape
        assert H == W == IMAGE_SIZE
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.max_pool(x)
        x = self.dropout(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        return x

    @staticmethod
    def add_to_argparse(parser):
        parser.add_argument("--conv_dim", type=int, default=CONV_DIM)
        parser.add_argument("--fc_dim", type=int, default=FC_DIM)
        return parser


import gym
import gymnasium

def gym_space_migration(gym_space: gym.Space) -> gymnasium.Space:
    if isinstance(gym_space, gym.spaces.Discrete):
        return gymnasium.spaces.Discrete(gym_space.n)

    elif isinstance(gym_space, gym.spaces.Box):
        return gymnasium.spaces.Box(
            low=gym_space.low,
            high=gym_space.high,
            shape=gym_space.shape,
            dtype=gym_space.dtype
        )

    elif isinstance(gym_space, gym.spaces.Dict):
        migrated_dict = {key: gym_space_migration(space) for key, space in gym_space.spaces.items()}
        return gymnasium.spaces.Dict(migrated_dict)

    elif isinstance(gym_space, gym.spaces.MultiDiscrete):
        return gymnasium.spaces.MultiDiscrete(gym_space.nvec)

    elif isinstance(gym_space, gym.spaces.Tuple):
        migrated_spaces = tuple(gym_space_migration(space) for space in gym_space.spaces)
        return gymnasium.spaces.Tuple(migrated_spaces)

def pretty_print_configs(config):
    return {
        "env": config.env,
        "framework": config.framework_str,
        "gamma": config.gamma
    }


import keras
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
# Dense 全连接层，Activation 激活函数
from keras.layers import Dense, Activation
# 优化器
from keras.optimizers import SGD

def replace_char(string, char, index):
    string = list(string)
    string[index] = char
    return ''.join(string)

# 随机生成200个随机点
x_data = np.linspace(-0.5, 0.5, 200)
noise = np.random.normal(0, 0.02, x_data.shape)
# square 是平方
y_data = np.square(x_data) + noise

plt.scatter(x_data, y_data)
plt.show()

# 构建一个顺序模型
model = Sequential()

# 在模型中添加一个全连接层
# 1-10-1
# 构建一个隐藏层输入出是10个神经元，输入是一个神经元
model.add(Dense(units=10, input_dim=1))

# 激活函数默认下没有，需要指定激活函数，导入激活函数包,tanh大多数时候比sigmod函数好用
model.add(Activation('tanh'))
# 下面这样也行
# model.add(Dense(units=10, input_dim=1,activate='relu'))

# 构建输出层，上一层是10个神经元，这里不用指定，只指定输出即可
model.add(Dense(units=1))

# 添加激活函数
model.add(Activation('tanh'))

# sgd是随机梯度下降默认学习率很小，大概是0.01，loss均平方误差
# model.compile(optimizer='sgd', loss='mse')

# 修改学习率需要导入kera.optimizers impprt SGD
# 定义优化算法
sgd = SGD(lr=0.3)

# 将优化器装入神经网络中(上面的compile注释掉)
model.compile(optimizer=sgd, loss='mse')

progress = '[..............................]'
for step in range(3000):
    cost = model.train_on_batch(x_data, y_data)
    if step % 100 == 0:
        progress = replace_char(progress, "=", progress.index("."))
        print(progress)
        print('cost', cost)
W, b = model.layers[0].get_weights()
print('W', W, 'b', b)

# x_data 输入网络中，得到预测值
y_pred = model.predict(x_data)

# 显示随机点
# plt.scatter(x_data, y_data, c='r')
plt.plot(x_data, y_pred, "r-")
plt.show()


import numpy as np
from random import *
from fractions import Fraction
import random

def fill(banks,bankNum):
    borrowers = 0

    # go through each row/bank
    for x in range (0, bankNum):
        # input user for filling banks
        line = input()

        # split input
        splitLine = line.split()
        splitLine = splitLine
                    # split_line should hold the line input as an array with each number splitted up

        # parse input
        banks[x][x] = float(splitLine[0])
        borrowers = int(splitLine[1])
        for y in range (2,(2+borrowers)+1):
            if (y%2 == 0):
                banks[x][int(splitLine[y])] = float(splitLine[y+1])
                    # current bank part of the matrix should be filled out with the correct values

    return 0

def judge(banks,bankNum,min):
    unsafe = []
    checker = 1

    while(checker > 0 ):
        checker = 0
        for x in range(0,bankNum):
            if x not in unsafe:
                if(sum(banks[x,:]) < min):
                    unsafe.append(x)
                    banks[x,:] = 0
                    banks[:,x] = 0
                    checker = checker+1
            else:
                continue

    print("unsafe banks: ")
    print(unsafe)

    return 0

def main():
    bankNum = 0
    min = 0

    # ask use how many banks there are and what the minimum asset is
    bankNum, min = input("How many banks are there and what is the minimum total asset?: ").split()
    bankNum = int(bankNum)
    min = int(min)
                # number of banks and minimum total asset is set

    # make matrix
    banks = ((bankNum,bankNum))
    banks = np.zeros(banks)
                # banks should have the proper size

    # banks lend money to each other
    fill(banks,bankNum)
                # banks should have all of their respective values (loaned and borrowed)

    # judge whether each bank is safe
    judge(banks,bankNum, min)
    return 0

# calls main()
if __name__ == "__main__":
    main()


import sys
from collections import namedtuple, OrderedDict
import atexit
import logging
from absl import flags
import gin
import numpy as np
from pysc2.lib.features import parse_agent_interface_format, SCREEN_FEATURES, MINIMAP_FEATURES, Features, FeatureType
from pysc2.env.environment import StepType
from pysc2.lib.actions import FunctionCall, FUNCTIONS
from pysc2.lib.remote_controller import RequestError

EnvironmentSpec = namedtuple('EnvironmentSpec', ['action_spec', 'observation_spec'])
ObservationSpec = namedtuple('ObservationSpec', ['id', 'shape', 'is_spatial', 'features'])
FeatureSpec = namedtuple('FeatureSpec', ['index', 'scale', 'is_categorical'])
ActionSpec = namedtuple('ActionSpec', ['id', 'sizes', 'obs_space', 'args_mask'])

@gin.configurable
class SC2EnvironmentConfig:
    def __init__(self, map_name=gin.REQUIRED, screen_size=16, minimap_size=16, function_set='minigames', visualize=False):
        self.map_name = map_name
        self.screen_size = screen_size
        self.minimap_size = minimap_size
        self.function_set = function_set
        self.visualize = visualize

class SC2Environment:
    def __init__(self, config: SC2EnvironmentConfig):
        self._aif = parse_agent_interface_format(feature_screen=config.screen_size, feature_minimap=config.minimap_size)
        self._visualize = config.visualize

        if config.function_set == 'all':
            self._func_ids = [f.id for f in FUNCTIONS]
        elif config.function_set == 'minigames':
            self._func_ids = [0, 1, 2, 3, 4, 6, 7, 12, 13, 42, 44, 50, 91, 183, 234, 309, 331, 332, 333, 334, 451, 452, 490]
        else:
            raise ValueError

        sc2_features = Features(agent_interface_format=self._aif)
        sc2_action_spec = sc2_features.action_spec()
        sc2_obs_spec = sc2_features.observation_spec()

        fn_args_mask = np.zeros((len(self._func_ids), len(sc2_action_spec.types) + 1), dtype=np.bool)
        fn_args_mask[:, 0] = 1
        for i, func_id in enumerate(self._func_ids):
            used_args = [a.id + 1 for a in FUNCTIONS[func_id].args]
            fn_args_mask[i, used_args] = 1
        action_spec = [('function_id', ActionSpec(0, (len(self._func_ids),), None, fn_args_mask))]
        for t in sc2_action_spec.types:
            if t.name == 'screen' or t.name == 'screen2':
                space = 'screen'
            elif t.name == 'minimap':
                space = 'minimap'
            else:
                space = None

            action_spec.append((t.name, ActionSpec(len(action_spec), t.sizes, space, None)))
        action_spec = OrderedDict(action_spec)

        def feature_spec(features):
            return [FeatureSpec(f.index, f.scale, f.type == FeatureType.CATEGORICAL) for f in features]

        obs_spec = OrderedDict([
            ('screen', ObservationSpec(0, sc2_obs_spec['feature_screen'], True, feature_spec(SCREEN_FEATURES))),
            ('minimap', ObservationSpec(1, sc2_obs_spec['feature_minimap'], True, feature_spec(MINIMAP_FEATURES))),
            ('available_actions', ObservationSpec(2, (len(self._func_ids),), False, None)),
            ('player', ObservationSpec(3, sc2_obs_spec['player'], False, None))
        ])

        self.spec = EnvironmentSpec(action_spec, obs_spec)

        from pysc2.env.sc2_env import SC2Env, Agent, Race

        if not flags.FLAGS.is_parsed():
            flags.FLAGS(sys.argv)

        num_retries = 3
        while True:
            try:
                self._env = SC2Env(map_name=config.map_name, agent_interface_format=self._aif, players=[
                    Agent(Race.protoss)
                ], visualize=self._visualize)

                break
            except RequestError:
                num_retries -= 1
                logging.error('SC2Env creation failed, {} retries remaining'.format(num_retries))
                if num_retries <= 0:
                    raise

        atexit.register(self._env.close)

    def close(self):
        self._env.close()
        atexit.unregister(self._env.close)

    def reset(self):
        obs, rewards, done = self._wrap_obs(self._env.reset())
        return obs

    def step(self, actions):
        sc2_actions = [self._actions_to_sc2(a) for a in actions]

        obs = self._env.step(sc2_actions)

        obs, rewards, done = self._wrap_obs(obs)
        if done:
            obs = self.reset()

        return obs, rewards, done

    def _wrap_obs(self, obs):
        def wrap(o):
            available_actions = np.zeros(self.spec.observation_spec['available_actions'].shape, dtype=np.int32)
            func_ids = [i for i, func_id in enumerate(self._func_ids) if func_id in o.observation['available_actions']] # TODO: this is too slow when using all function ids
            available_actions[func_ids] = 1

            return {
                'screen': np.asarray(o.observation['feature_screen']),
                'minimap': np.asarray(o.observation['feature_minimap']),
                'available_actions': available_actions,
                'player': np.asarray(o.observation['player'])
            }

        return [wrap(o) for o in obs], [o.reward for o in obs], obs[0].step_type == StepType.LAST

    def _actions_to_sc2(self, actions):
        def convert_arg(value, spec):
            if len(spec.sizes) == 2:
                value = np.unravel_index(value, spec.sizes)
                value = np.flip(value)
                return list(value)
            else:
                return [value]
        function = self._func_ids[actions['function_id']]
        args = [
            convert_arg(actions[arg.name].item(), self.spec.action_spec[arg.name])
            for arg in FUNCTIONS[function].args
        ]

        return FunctionCall(function, args)

    def _palettes(self):
        feat_palettes = [[None] * len(s.features) for s in env_spec.spaces]
        for s in env_spec.spaces:
            for f in s.features:
                palette = f.palette
                if len(palette) < f.scale:
                    palette = np.append(f.palette, [[255, 0, 255] * (f.scale - len(f.palette))], axis=0)
                feat_palettes[s.index][f.index] = tf.constant(palette, dtype=tf.uint8,
                                                              name='{}_{}_palette'.format(s.name, f.name))


"""
 HTTP client tool for Svom

 Retries requests with power law delays and a max tries limit

 @author: henri.louvin@cea.fr
"""

import threading

import time

import requests

import asyncio
import aiohttp
import signal

# Log
import logging
log = logging.getLogger('http_client')

class HttpClient(threading.Thread):
    """
    Threaded asyncio NATS client
    """
    def __init__(self, _http_client, loop):
        """Init asyncio thread"""
        threading.Thread.__init__(self)
        self._http_client = _http_client
        self.loop = loop

    # called by thread.start()
    def run(self):
        """Start asyncio loop"""
        if not self.loop.is_running():
            log.debug("Starting async event loop")
            self.loop.run_forever()

class HttpIo:
    """
    (A)synchronous HTTP client
    """
    def __init__(self, server_url, max_tries=5, backoff_factor=1, asynchronous=False, loop=None): #pylint: disable=R0913
        self.server_url = server_url.strip('/')
        self.max_tries = max_tries
        self.backoff_factor = backoff_factor
        self.loop = None
        self.async_session = None
        self.asynchronous = asynchronous
        if asynchronous is True:
            self.loop = loop if loop is not None else asyncio.get_event_loop()
            self.async_session = self._get_client(self.loop)
            signal.signal(signal.SIGINT, self.sigint_handler)
            async_client = HttpClient(self.async_session, self.loop)
            async_client.start()

    def _get_client(self, loop):
        return aiohttp.ClientSession(loop=loop)

    def _compute_delay(self, tried):
        """ calculate delay in seconds as a power law """
        delay = self.backoff_factor * (2 ** (tried-1))
        return delay

    def _sync_request(self, req, url, **kwargs):
        """ Synchronous {req} request to {url} with data {kwargs}"""
        tried = 0
        response = None
        success = False
        log.info('%s request to %s: %s', req, url, kwargs)
        while success is False:
            if req == 'GET':
                response = requests.get(url, **kwargs)
            elif req == 'POST':
                response = requests.post(url, **kwargs)
            elif req == 'PUT':
                response = requests.put(url, **kwargs)
            else:
                exc = f"_sync_request() only handles 'GET', 'POST' and 'PUT' requests, not '{req}'"
                raise ValueError(exc)
            tried += 1
            # retrieve response.json()
            json_resp = {}
            try:
                json_resp = response.json()
            except Exception:
                pass
            log.debug('Server %s response (%s): %s', req, response.status_code, json_resp)
            # accept all responses with status 2xx
            if int(response.status_code/100) == 2:
                log.info('%s request to %s successfull (%s)', req, url, response.status_code)
                success = True
            elif tried < self.max_tries:
                delay = self._compute_delay(tried)
                log.warning('%s request to %s failed (%s). Trying again in %ss',
                            req, url, response.status_code, delay)
                time.sleep(delay)
                log.debug('%s request (%sth try) to %s: %s', req, tried+1, url, kwargs)
            else:
                log.warning('%s request to %s failed (%s)', req, url, response.status_code)
                log.error('Last server %s response (%s): %s', req, response.status_code, json_resp)
                log.error('%s request to %s failed %s times. Aborting', req, url, self.max_tries)
                break
        # override response.json()
        response.json = lambda: json_resp
        return response

    async def _async_request(self, req, url, tried, **kwargs):
        """ Asynchronous request to {url} with data {kwatgs}"""
        # delay request if needed
        if tried > 0:
            delay = self._compute_delay(tried)
            await asyncio.sleep(delay)
            log.debug('%s request (async, %sth try) to %s: %s', req, tried+1, url, kwargs)
        else:
            log.info('%s request (async) to %s: %s', req, url, kwargs)
        tried += 1
        try:
            response = None
            if req == 'GET':
                response = await self.async_session.get(url, **kwargs)
            elif req == 'POST':
                response = await self.async_session.post(url, **kwargs)
            elif req == 'PUT':
                response = await self.async_session.put(url, **kwargs)
            else:
                exc = f"_async_request() only handles 'GET', 'POST' and 'PUT' requests, not '{req}'"
                raise ValueError(exc)
            # retrieve response.json()
            json_resp = {}
            try:
                json_resp = await response.json()
            except Exception:
                pass
            log.debug('Server %s response (%s): %s', req, response.status, json_resp)
            # accept all responses with status 2xx
            if int(response.status/100) == 2:
                log.info('%s request (async) to %s successfull (%s)', req, url, response.status)
                # override response.json()
                response.json = lambda: json_resp
                response.status_code = response.status
            elif tried < self.max_tries:
                delay = self._compute_delay(tried)
                log.warning('%s request (async) to %s failed (%s). Trying again in %ss',
                            req, url, response.status, delay)
                response = await self._async_request(req, url, tried, **kwargs)
            else:
                log.warning('%s request (async) to %s failed (%s)', req, url, response.status)
                log.error('Last server %s response (%s): %s', req, response.status, json_resp)
                log.error('%s request (async) to %s failed %s times. Aborting',
                          req, url, self.max_tries, response.status)

                # override response.json()
                response.json = lambda: json_resp
                response.status_code = response.status
        except Exception as exc:
            log.error('Exception caught: %s', exc)
            delay = self._compute_delay(tried)
            log.error('%s request (async) to %s failed. Aborting', req, url)
            raise
        return response

    def get(self, endpoint='/', **kwargs):
        """ GET request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return self._sync_request('GET', url, **kwargs)

    def post(self, endpoint='/', **kwargs):
        """ POST request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return self._sync_request('POST', url, **kwargs)

    def put(self, endpoint='/', **kwargs):
        """ PUT request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return self._sync_request('PUT', url, **kwargs)

    async def async_get(self, endpoint='/', **kwargs):
        """ GET request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return await self._async_request('GET', url, tried=0, **kwargs)

    async def async_post(self, endpoint='/', **kwargs):
        """ POST request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return await self._async_request('POST', url, tried=0, **kwargs)

    async def async_put(self, endpoint='/', **kwargs):
        """ PUT request to endpoint {endpoint} with json data {data}"""
        url = f"{self.server_url}/{endpoint.strip('/')}"
        return await self._async_request('PUT', url, tried=0, **kwargs)

    def stop(self):
        """ close http session if asynchronous """
        if self.asynchronous is False:
            return
        log.debug('Stopping async client...')
        log.debug('Cancelling pending tasks...')
        # Cancel pending tasks
        for task in asyncio.Task.all_tasks():
            task.cancel()
        # Stop HTTP session properly
        future = asyncio.run_coroutine_threadsafe(self.async_session.close(),
                                                  loop=self.loop)
        # Wait for HTTP session to close then stop async loop
        try:
            log.debug('Waiting for aiohttp.ClientSession to close...')
            future.result()
            log.debug('Done. Stopping async event loop')
            self.loop.call_soon_threadsafe(self.loop.stop)
        except Exception as err:
            log.error(err)

    def sigint_handler(self, signum, frame):
        """
        stops gracefully, restore default signal handling
        and raises KeyboardInterrupt
        """
        self.stop()
        signal.signal(signal.SIGINT, signal.SIG_DFL)
        raise KeyboardInterrupt


import yaml
import os

import random
import numpy as np
import pandas as pd
from tqdm import tqdm

import torch
import torchvision

from torch.utils.tensorboard import SummaryWriter

from utils import get_root

from transforms import Resize, ToTensor, BatchRicianNoise, BatchRandomErasing
from dataset import DenoisingDataset

from models import DnCNN

from metrics import PSNR, SSIM, FSIM
from losses import SIMMLoss, L1Loss, CombinedLoss

# for printing
torch.set_printoptions(precision=2)

# for reproducibility
seed = 0

random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)

if torch.cuda.is_available():
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def get_params(root):
    with open(os.path.join(root, "configs.yaml"), "r") as config_file:
        configs = yaml.load(config_file, Loader=yaml.FullLoader)
    params = {'train_data': configs['paths']['data']['train_data'],
              'dataset_table_path': configs['paths']['dataset_table'],
              'log_dir': configs['paths']['log_dir']}

    for param in params.keys():
        params[param] = os.path.join(root, params[param])

    params.update({'image_size': tuple(map(int, configs['data_parameters']['image_size'].split(', '))),
                   'batch_size': int(configs['data_parameters']['batch_size'])})

    params.update({'num_features': int(configs['model_parameters']['DnCNN']['num_features']),
                   'num_layers': int(configs['model_parameters']['DnCNN']['num_layers'])})

    params.update({'fourier_layer': configs['model_parameters']['Fourier']['fourier_layer']})

    params.update({'lr': float(configs['train_parameters']['lr']),
                   'n_epochs': int(configs['train_parameters']['epochs'])})

    return params

def make_dataset_table(data_path, csv_file_path):
    image_names = sorted([name for name in os.listdir(os.path.join(data_path, 'Cropped images'))])
    data = []

    print('dataset csv table creating...')
    for image_name in tqdm(image_names):
        image_path = os.path.join(data_path, 'Cropped images', image_name)
        data.append(np.array([image_path]))

    pd.DataFrame(np.vstack(data), columns=['image']).to_csv(csv_file_path, index=False)

def train_val_split(csv_file_path, val_size=0.2):
    dataset = pd.read_csv(csv_file_path)

    test_number = int(len(dataset) * val_size) + 1
    train_number = len(dataset) - test_number
    phase = ['train'] * train_number + ['val'] * test_number
    random.Random(1).shuffle(phase)

    pd.concat([dataset[['image']],
               pd.DataFrame(phase, columns=['phase'])],
              axis=1).to_csv(csv_file_path, index=False)

def setup_experiment(title, params, log_dir):
    model_name = [title, 'fourier', params['fourier_layer']]
    # model_name.extend(['lr', str(params['lr'])])
    model_name = '_'.join(model_name)

    writer = SummaryWriter(log_dir=os.path.join(log_dir, model_name))
    best_model_path = f"{model_name}.best.pth"

    return writer, model_name, best_model_path

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

def collate_transform(batch_transform=None):
    def collate(batch):
        collated = torch.utils.data.dataloader.default_collate(batch)
        if batch_transform is not None:
            collated = batch_transform(collated)
        return collated
    return collate

def run_epoch(model, iterator,
              criterion, optimizer,
              metrics,
              phase='train', epoch=0,
              device='cpu', writer=None):
    is_train = (phase == 'train')
    if is_train:
        model.train()
    else:
        model.eval()

    epoch_loss = 0.0
    epoch_metrics = {metric_name: 0.0 for metric_name in metrics.keys()}

    with torch.set_grad_enabled(is_train):
        for i, (images, masks) in enumerate(tqdm(iterator)):
            images, masks = images.to(device), masks.to(device)

            cleaned_images = model(images)

            loss = criterion(cleaned_images, masks)

            if is_train:
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

            epoch_loss += loss.item()
            for metric_name in epoch_metrics.keys():
                epoch_metrics[metric_name] += metrics[metric_name](cleaned_images.detach(), masks)

        epoch_loss = epoch_loss / len(iterator)
        for metric_name in epoch_metrics.keys():
            epoch_metrics[metric_name] = epoch_metrics[metric_name] / len(iterator)

        if writer is not None:
            writer.add_scalar(f"loss_epoch/{phase}", epoch_loss, epoch)
            for metric_name in epoch_metrics.keys():
                writer.add_scalar(f"metric_epoch/{metric_name}/{phase}", epoch_metrics[metric_name], epoch)

        return epoch_loss, epoch_metrics

def train(model,
          train_dataloader, val_dataloader,
          criterion,
          optimizer, scheduler,
          metrics,
          n_epochs,
          device,
          writer,
          best_model_path):
    best_val_loss = float('+inf')
    for epoch in range(n_epochs):
        train_loss, train_metrics = run_epoch(model, train_dataloader,
                                              criterion, optimizer,
                                              metrics,
                                              phase='train', epoch=epoch,
                                              device=device, writer=writer)
        val_loss, val_metrics = run_epoch(model, val_dataloader,
                                          criterion, None,
                                          metrics,
                                          phase='val', epoch=epoch,
                                          device=device, writer=writer)
        if scheduler is not None:
            scheduler.step(val_loss)

        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(model.state_dict(), best_model_path)

        print(f'Epoch: {epoch + 1:02}')

        metrics_output = ' | '.join([metric_name + ': ' +
                                     "{:.2f}".format(train_metrics[metric_name]) for metric_name in train_metrics.keys()])
        print(f'\tTrain Loss: {train_loss:.2f} | Train Metrics: ' + metrics_output)

        metrics_output = ' | '.join([metric_name + ': ' +
                                     "{:.2f}".format(val_metrics[metric_name]) for metric_name in val_metrics.keys()])
        print(f'\t  Val Loss: {val_loss:.2f} |   Val Metrics: ' + metrics_output)

def main():
    root = get_root()
    params = get_params(root)
    train_data_path, dataset_table_path, log_dir = (params['train_data'],
                                                    params['dataset_table_path'],
                                                    params['log_dir'])
    image_size, batch_size = (params['image_size'],
                              params['batch_size'])
    lr, n_epochs = (params['lr'],
                    params['n_epochs'])

    make_dataset_table(train_data_path, dataset_table_path)
    train_val_split(dataset_table_path)
    dataset = pd.read_csv(dataset_table_path)

    pre_transforms = torchvision.transforms.Compose([Resize(size=image_size), ToTensor()])
    train_batch_transforms = BatchRicianNoise()
    # train_batch_transforms = BatchRandomErasing()

    train_dataset = DenoisingDataset(dataset=dataset[dataset['phase'] == 'train'],
                                     transform=pre_transforms)

    train_collate = collate_transform(train_batch_transforms)
    train_dataloader = torch.utils.data.DataLoader(dataset=train_dataset,
                                                   batch_size=batch_size,
                                                   shuffle=True,
                                                   collate_fn=train_collate)

    val_batch_transforms = BatchRicianNoise()
    # val_batch_transforms = BatchRandomErasing()

    val_dataset = DenoisingDataset(dataset=dataset[dataset['phase'] == 'val'],
                                   transform=pre_transforms)

    val_collate = collate_transform(val_batch_transforms)
    val_dataloader = torch.utils.data.DataLoader(dataset=val_dataset,
                                                 collate_fn=val_collate)

    fourier_params = None
    if params['fourier_layer'] != 'None':
        fourier_params = {'fourier_layer': params['fourier_layer']}

    num_features, num_layers = params['num_features'], params['num_layers']
    model = DnCNN(n_channels=1,
                  num_features=num_features, num_layers=num_layers,
                  image_size=image_size, fourier_params=fourier_params).to(device)

    writer, model_name, best_model_path = setup_experiment(model.__class__.__name__, params, log_dir)
    best_model_path = os.path.join(root, best_model_path)
    print(f"Model name: {model_name}")
    print(f"Model has {count_parameters(model):,} trainable parameters")
    print()

    criterion = CombinedLoss([SIMMLoss(multiscale=True), L1Loss()], [0.8, 0.2])
    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=5)
    metrics = {'PSNR': PSNR(), 'SSIM': SSIM(multiscale=False), 'FSIM': FSIM()}

    print("To see the learning process, use command in the new terminal:\ntensorboard --logdir <path to log directory>")
    print()
    train(model,
          train_dataloader, val_dataloader,
          criterion,
          optimizer, scheduler,
          metrics,
          n_epochs,
          device,
          writer,
          best_model_path)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

from builtins import object
from PyQt5 import QtCore, QtGui, QtWidgets

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_qgsnewhttpconnectionbase(object):
    def setupUi(self, qgsnewhttpconnectionbase):
        qgsnewhttpconnectionbase.setObjectName(_fromUtf8("qgsnewhttpconnectionbase"))
        qgsnewhttpconnectionbase.resize(642, 153)
        self.buttonBox = QtWidgets.QDialogButtonBox(qgsnewhttpconnectionbase)
        self.buttonBox.setGeometry(QtCore.QRect(280, 110, 341, 32))
        self.buttonBox.setFocusPolicy(QtCore.Qt.NoFocus)
        self.buttonBox.setOrientation(QtCore.Qt.Horizontal)
        self.buttonBox.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok)
        self.buttonBox.setObjectName(_fromUtf8("buttonBox"))
        self.label_NewSrvName = QtWidgets.QLabel(qgsnewhttpconnectionbase)
        self.label_NewSrvName.setGeometry(QtCore.QRect(20, 33, 91, 17))
        self.label_NewSrvName.setObjectName(_fromUtf8("label_NewSrvName"))
        self.label_NewSrvUrl = QtWidgets.QLabel(qgsnewhttpconnectionbase)
        self.label_NewSrvUrl.setGeometry(QtCore.QRect(20, 75, 91, 17))
        self.label_NewSrvUrl.setObjectName(_fromUtf8("label_NewSrvUrl"))
        self.txt_NewSrvName = QtWidgets.QLineEdit(qgsnewhttpconnectionbase)
        self.txt_NewSrvName.setEnabled(True)
        self.txt_NewSrvName.setGeometry(QtCore.QRect(120, 30, 501, 27))
        self.txt_NewSrvName.setCursor(QtGui.QCursor(QtCore.Qt.IBeamCursor))
        self.txt_NewSrvName.setFocusPolicy(QtCore.Qt.ClickFocus)
        self.txt_NewSrvName.setObjectName(_fromUtf8("txt_NewSrvName"))
        self.txt_NewSrvUrl = QtWidgets.QLineEdit(qgsnewhttpconnectionbase)
        self.txt_NewSrvUrl.setGeometry(QtCore.QRect(120, 70, 501, 27))
        self.txt_NewSrvUrl.setObjectName(_fromUtf8("txt_NewSrvUrl"))

        self.retranslateUi(qgsnewhttpconnectionbase)
        self.buttonBox.accepted.connect(qgsnewhttpconnectionbase.accept)
        self.buttonBox.rejected.connect(qgsnewhttpconnectionbase.reject)
        QtCore.QMetaObject.connectSlotsByName(qgsnewhttpconnectionbase)

    def retranslateUi(self, qgsnewhttpconnectionbase):
        qgsnewhttpconnectionbase.setWindowTitle(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "New WCPS Server ", None))
        self.label_NewSrvName.setText(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "Server Name", None))
        self.label_NewSrvUrl.setText(QtWidgets.QApplication.translate("qgsnewhttpconnectionbase", "Server URL", None))


# -*- coding: utf-8 -*-

from mrjob.job import MRJob
from mrjob.protocol import JSONProtocol, RawValueProtocol
from mrjob.step import MRStep
import numpy as np

######################## Helper Methods and Classes  ##########################

def cholesky_solution_linear_regression(x_t_x,x_t_y):
    '''
    Finds parameters of regression through Cholesky decomposition,
    given sample covariance of explanatory variables and covariance
    between explanatory variable and dependent variable.

    Paramaters:
    -----------
    x_t_x    - numpy array of size 'm x m', represents sample covariance of explanatory variables
    x_t_y    - numpy array of size 'm x 1', represent covariance between expalanatory and dependent variable

    Output:
    -------
    Theta   - list of size m, represents values of coefficients

    '''
    # L*L.T*Theta = x_t_y
    L = np.linalg.cholesky(x_t_x)
    #  solve L*z = x_t_y
    z = np.linalg.solve(L,x_t_y)
    #  solve L.T*Theta = z
    theta = np.linalg.solve(np.transpose(L),z)
    return theta

class DimensionMismatchError(Exception):

    def __init__(self,expected,observed):
        self.exp = expected
        self.obs = observed

    def __str__(self):
        err = "Expected number of dimensions: "+str(self.exp)+", observed: "+str(self.obs)
        return err

############################## Map Reduce Job #################################

class LinearRegressionTS(MRJob):
    '''
    Calculates sample covariance matix of explanatory variables (x_t_x) and
    vector of covariances between dependent variable expanatory variables (x_t_y)
    in single map reduce pass and then uses cholesky decomposition to
    obtain values of regression parameters.

    Important!!! Since final computations are performed on single reducer,
    assumption is that dimensionality of data is relatively small i.e. input
    matrix is tall and skinny.

    Input File:
    -----------

          Extract relevant features from input line by changing extract_variables
          method. You can add features for non-linear models (like x^2 or exp(x)).
          Current code assumes following input line format:

          input line = <dependent variable>, <feature_1>,...,<feature_n>

    Options:
    -----------

          -- dimension  - (int)  number of explanatory variables
          -- bias       - (bool) if True regression wil include bias term

    Output:
    -----------
          json-encoded list of parameters
    '''

    INPUT_PROTOCOL = RawValueProtocol

    INTERNAL_PROTOCOL = JSONProtocol

    OUTPUT_PROTOCOL = RawValueProtocol

    def __init__(self,*args, **kwargs):
        super(LinearRegressionTS, self).__init__(*args, **kwargs)
        n = self.options.dimension
        self.x_t_x  = np.zeros([n,n])
        self.x_t_y  = np.zeros(n)
        self.counts = 0

    #--------------------------- feature extraction --------------------------#

    def extract_variables(self,line):
        ''' (str)--(float,[float,float,float...])
        Extracts set of relevant features. (Needs to be rewriten depending
        on file input structure)
        '''
        data = [float(e) for e in line.strip().split(",")]
        y,features = data[0],data[1:]
        return (y,features)

    #---------------------------- Options ------------------------------------#

    def configure_options(self):
        ''' Additional options'''
        super(LinearRegressionTS,self).configure_options()
        self.add_passthrough_option("--dimension",
                                    type = int,
                                    help = "Number of explanatory variables (do not count bias term)")
        self.add_passthrough_option("--bias",
                                    type = str, # (got error when tried to define bool) ???
                                    help = "Bias term, bias not included if anything other than 'True' ",
                                    default = "True")

    def load_options(self,args):
        ''' Loads and checks whether options are provided'''
        super(LinearRegressionTS,self).load_options(args)
        if self.options.dimension is None:
            self.option_parser.error("You should define number of explanatory variables")
        else:
            self.dim = self.options.dimension

    #------------------------ map-reduce steps -------------------------------#

    def mapper_lr(self,_,line):
        '''
        Calculates x_t_x and x_t_y for data processed by each mapper
        '''
        y,features = self.extract_variables(line)
        if len(features) != self.dim:
            raise DimensionMismatchError(self.dim,len(features))
        if self.options.bias is "True":
            features.append(1.0)
        x = np.array(features)
        self.x_t_x  += np.outer(x, x)
        self.x_t_y  += y*x
        self.counts += 1

    def mapper_lr_final(self):
        '''
        Transforms numpy arrays x_t_x and x_t_y into json-encodable list format
        and sends to reducer
        '''
        yield 1,("x_t_x",  [list(row) for row in self.x_t_x])
        yield 1,("x_t_y",  [xy for xy in self.x_t_y])
        yield 1,("counts", self.counts)

    def reducer_lr(self,key,values):
        '''
        Aggregates results produced by each mapper and obtains x_t_x and x_t_y
        for all data, then using cholesky decomposition obtains parameters of
        linear regression.
        '''
        n = self.dim
        observations = 0
        x_t_x = np.zeros([n,n]); x_t_y = np.zeros(n)
        for val in values:
            if val[0]=="x_t_x":
                x_t_x           +=   np.array(val[1])
            elif val[0]=="x_t_y":
                x_t_y           +=   np.array(val[1])
            elif val[0]=="counts":
                observations    +=   val[1]
        betas = cholesky_solution_linear_regression(x_t_x,x_t_y)
        yield None,[e for e in betas]

    def steps(self):
        '''Defines map-reduce steps '''
        return [MRStep(mapper       = self.mapper_lr,
                       mapper_final = self.mapper_lr_final,
                       reducer      = self.reducer_lr)]

if __name__=="__main__":
    LinearRegressionTS.run()


import requests
import re
import pymysql
from lxml import etree

header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}
reques = requests.get('http://www.open.com.cn/encrollregulation-3333.html', headers=header)
reques.encoding = 'utf-8'
c2 = reques.text
# print(c2)
yb = re.compile('
(.*?)
', re.S)
yb2 = re.findall(yb, c2)
        # print(yp2)
for b in yb2:
        if '报名办法' in b:
            ybj = re.compile('
.*?
(.*?)<h.*?>', re.S)
            ybs = re.findall(ybj, c2)
            for ys in ybs:
                yjsssb = re.sub('奥鹏', '舟炬', ys)
                yjss1 = re.sub('http://www.open.com.cn', 'http://www.zhoujuedu.com', yjsssb)
                yjss2 = re.sub(' |||||||
', '', yjss1)
                yjss3 = re.compile('
(.*?)
', re.S)
                yjss4 = re.findall(yjss3, yjss2)
                yjss5 = ','.join(yjss4)
                print(yjss5)
        elif '入学资格' in b:
            ybj2 = re.compile('
.*?
(.*?)<h.*?>', re.S)
            ybs2 = re.findall(ybj2, c2)
            for ys in ybs2:
                yjsssb2 = re.sub('奥鹏', '舟炬', ys)
                yjss12 = re.sub('http://www.open.com.cn', 'http://www.zhoujuedu.com', yjsssb2)
                yjss22 = re.sub(' |||||||
', '', yjss12)
                yjss32 = re.compile('
(.*?)
', re.S)
                yjss42 = re.findall(yjss32, yjss22)
                yjss52 = ','.join(yjss42)
                print(yjss52)
        elif '报名办法' or '入学资格' not in b:
            s = []
            yj2 = re.compile('
.*?
(.*?)<h.*?>', re.S)
            yjs2 = re.findall(yj2, c2)  # 获取所有p标签
            for ys2 in yjs2:
                yjs0 = re.sub('奥鹏','舟炬',ys2)
                yjs6 = re.sub('
', '', yjs0)
                yjs4 = re.compile('
(.*?)
', re.S)
                yjs5 = re.findall(yjs4, yjs6)  # 获取所有p标签的内容，并拼接为字符串
                yjs3 = ','.join(yjs5)
                s.append(yjs3)
            yjj2 = re.compile('
.*?
(.*?)<h.*?>', re.S)
            yjjs2 = re.findall(yjj2, c2)  # 获取所有p标签
            for ysj2 in yjjs2:
                yjs01 = re.sub('奥鹏','舟炬',ysj2)
                yjs61 = re.sub('
', '', yjs01)
                yjs41 = re.compile('
(.*?)
', re.S)
                yjs51 = re.findall(yjs41, yjs61)  # 获取所有p标签的内容，并拼接为字符串
                yjs31 = ','.join(yjs51)
                s.append(yjs31)
            s2 = ','.join(s)
            print(s2)


from server import *
from functionGen import *
from trigger import *
from spi import *
import math
import time
import queue
import threading
import sys

class MiddleWare:

    def __init__(self):
        self.points = 60
        self.living = True

        self.server = Server()
        self.spi = Spi(self.points)
        self.queueSize = 20
        self.proccessQueue = queue.Queue(self.queueSize)
        self.oscWindow_1 = []

        self.trigger = Trigger(1,-5)

        #Thread to handle reading from SPI then writing to Server
        spiThread = threading.Thread(target = self.spiRead)
        spiThread.name = "SPI_Thread"
        spiThread.deamon = True    #Kill off on its own
        spiThread.start()

        #Thread to handle reading from Server then writing to SPI
        serverThread = threading.Thread(target = self.serverRead)
        serverThread.name = "SERVER_Thread"
        serverThread.deamon = True
        serverThread.start()

        print(threading.active_count())
        for thrd in threading.enumerate():
            if(thrd.isDaemon):
                print(thrd)

        while(self.living):
            x= 0
        print(threading.active_count())
        for thrd in threading.enumerate():
            if(thrd.isDaemon):
                print(thrd)

    def spiRead(self):
        """ reads from the spi then proccess the data before passing on to server.py

        """
        count = 0;
        while(self.living):
            if(not self.proccessQueue.empty()):
                message = self.proccessQueue.get()
                if(self.server.receiver_found):
                    self.server.addToSend(message)

            else:
                data = self.spi.read()
                # if count % 10 == 0:
                #     print(data)
                # count += 1
                if(str(data) != "empty"):
                    self.proccess(str(data))
                hz = 40000
                time.sleep(1/hz)
                # time.sleep(.25)
            # print(self.spi.dataQueue.qsize())

    def proccess(self, data):
        """ Perfroms neccesary calculation upon data
            data: var
                The data to be proccessed
        """
        proccessedData = float(data)
        if(self.proccessQueue.full()):
            self.proccessQueue.get()

        if(self.trigger.readyToSend):
            print("Send Data")
            trigger_window = list(self.trigger.window)
            tmp = []
            size = len(self.trigger.window)
            i = 0
            while i < size:
                tmp.append(str(trigger_window[i]))
                i = i + 1

            window = ', '.join(tmp)
            print("Data Points: "+str(len(tmp)))

            self.proccessQueue.put(window)

        if(proccessedData >= self.trigger.level and not self.trigger.record):
            self.trigger.beginWindow(float(proccessedData))

        if(self.trigger.record):
            self.trigger.addToWindow(proccessedData)

    def serverRead(self):
        while(self.living):
            userInput = self.server.recentMessage()
            if(userInput != "empty"):
                self.parseUser(userInput)

    def parseUser(self, text):
        splitText = text.split(",")
        if(splitText[0] == "fcnGen" and len(splitText) == 5):
            func = splitText[1]
            amp = splitText[2]
            freq = splitText[3]
            per = splitText[4]
            self.spi.funcGen.setValues(func, amp, freq, per)
            self.trigger.purge()
        elif(text == "GUI CLOSED"):
            print("QUITTING TIME")
            self.living = False
            self.server.active = False
            self.spi.stop = True

# class Spi:
#
#     def __init__(self,points):
#         self.points =points
#         self.pos = 30
#         #FunctionGen(Type, Amp, Freq, Peri)
#         self.funcGen = FunctionGen("sin", 1, 1, 1)
#
#     def read(self):
#         amp  = self.funcGen.amplitude
#         freq = self.funcGen.frequency
#         angle = self.pos * math.pi/180
#         voltage = math.sin(freq*angle) * (amp)
#
#         # voltage = math.sin(self.pos*2*math.pi/self.points) * (amp)
#         voltage = round(voltage, 3)
#         string = str(voltage)
#         self.pos = self.pos + 1
#         # if(self.pos==360):
#         #     self.pos = 0;
#         return string

middleware = MiddleWare()


from flask import Flask, make_response
from bson.json_util import dumps
from models import note
import json
import pymongo
import UserDAO
import PlacesDAO
import Place
import WeatherHour
import AlertInfo
import WeatherCamLink

app = Flask(__name__)
app.debug = True

connection_string = "mongodb://localhost"
connection = pymongo.MongoClient(connection_string)
database = connection.blog

users = UserDAO.UserDAO(database)
places = PlacesDAO.PlacesDAO(database)

# send in some test data
@app.route("/samples")
def addSamplePlaces():
    return make_response(open('tstData.json').read())

@app.route("/")
def app_endpoint():
    return make_response(open('templates/index.html').read())

@app.route("/notes")
def get_notes():
    notes = []
    list = [
        {'a': 1, 'b': 2},
        {'a': 5, 'b': 10}
    ]
    for x in xrange(0,10):
        n = note()
        n.title = "Title_" + str(x)
        notes.append(n.__dict__)
        return json.dumps(notes)

@app.route("/getPlace/<username>")
def get_place(username):
    return dumps(places.getPlaces(username));

@app.route("/getAlerts/<user_id>")
def get_alerts_for_user(user_id):
    pass

if __name__ == "__main__":
    app.run()


 # encoding: utf-8
import sys
import os
import io
from managermentFileForqiniu import managerfile
import logging
import datetime

##log
logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%Y %m %d  %H:%M:%S',filename='SyncOperation_main.log',filemode='a')
logging.info("/r/t")
##main
Sync_path='/home/pi/Documents/qiniuclouddir'
now=datetime.datetime.now()
strnow=now.strftime('%Y-%m-%d %H:%M:%S')
logging.info(strnow)
files=os.listdir(Sync_path)
if(len(files)==0):
	sys.exit()
	logging.info(strnow+"Do not find file in Sync_path.")
mf=managerfile()
ret,eof,info=mf.getfileslist(-1)
logging.info(info.status_code)
if(info.status_code!=200):
	logging.error(strnow+"qiniucloud connect fail.statusCode:"+info.status_code)
	sys.exit()
remotefile=ret.get("items")
logging.info(remotefile[0].get('key'))
for localfile in files:
	exist=0
	for file in remotefile:
		if(file.get("key")==localfile):
			exist=1
			break
	if(exist==0):
		result=mf.uploadfile(localfile,Sync_path+"/"+localfile)
		if(result.status_code==200):
			logging.info(strnow+"uplaod %s successed" % (localfile))
		else:
			logging.info(strnow+"upload %s fail" % (localfile))


# -*- coding: utf-8 -*-

#Za GUI
from turtle import Turtle, Screen
#Za komunikaciju
from socket import socket, AF_INET, SOCK_STREAM
#Za multi-threading
from _thread import start_new_thread
#Za odgodu-vrijeme
from time import sleep

def primi_thread(c):
    while True:
        y = c.recv(500).decode('UTF-8')
        paddle_b.sety(int(y))

def salji_funkciju(y):
    msg = 'p:' + str(y)
    c.send(msg.encode('UTF-8'))

def igra(c):
    #Rezultat
    rez_a = 0
    rez_b = 0

    #Čekanje 10 sekundi
    sleep(10)
    #Glavni dio igre
    while True:
        wind.update()

    	#Pomicanje loptice
        ball.setx(ball.xcor() + ball.dx)
        ball.sety(ball.ycor() + ball.dy)

        #Provjera granica
        if ball.ycor() > 140:
            ball.sety(140)
            ball.dy *= -1

        if ball.ycor() < -140:
            ball.sety(-140)
            ball.dy *= -1

        if ball.xcor() > 190:
            ball.goto(0, 0)
            ball.dx *= -1
            rez_a += 1

            info = "Igrač A: {}           Igrač B: {}".format(rez_a, rez_b)
            msg = 'r:' + str(rez_a) + ":" + str(rez_b)
            c.send(msg.encode('UTF-8'))

            pen.clear()
            pen.write(info, align="center", font=("Arial", 15, "normal"))

        if ball.xcor() < -190:
            ball.goto(0, 0)
            ball.dx *= -1
            rez_b += 1

            info = "Igrač A: {}           Igrač B: {}".format(rez_a, rez_b)
            msg = 'r:' + str(rez_a) + ":" + str(rez_b)
            c.send(msg.encode('UTF-8'))

            pen.clear()
            pen.write(info, align="center", font=("Arial", 15, "normal"))

        #Reket i loptica kolizija
        if ball.xcor() > 140 and ball.xcor() < 150 and ball.ycor() < paddle_b.ycor() + 40 and ball.ycor() > paddle_b.ycor() -40:
            ball.setx(140)
            ball.dx *= -1.2

        if ball.xcor() < -140 and ball.xcor() > -150 and ball.ycor() < paddle_a.ycor() + 40 and ball.ycor() > paddle_a.ycor() -40:
            ball.setx(-140)
            ball.dx /= -1.2

        msg = 'b:' + str(int(ball.xcor())) + ":" + str(int(ball.ycor()))
        print('Server Send Msg', msg)
        c.send(msg.encode('UTF-8'))

def paddle_a_up():
    y = paddle_a.ycor()
    y += 20
    paddle_a.sety(y)
    salji_funkciju(y)

def paddle_a_down():
    y = paddle_a.ycor()
    y -= 20
    paddle_a.sety(y)
    salji_funkciju(y)

wind = Screen()
wind.title("Ping Pong: Server")
wind.bgcolor("#222222")
wind.setup(width=450, height=350)
wind.tracer(0)

#Reket A
paddle_a = Turtle()
paddle_a.speed(0)
paddle_a.shape("square")
paddle_a.color("blue")
paddle_a.shapesize(stretch_wid=5, stretch_len=1)
paddle_a.penup()
paddle_a.goto(-150, 0)

#Reket B
paddle_b = Turtle()
paddle_b.speed(0)
paddle_b.shape("square")
paddle_b.color("red")
paddle_b.shapesize(stretch_wid=5, stretch_len=1)
paddle_b.penup()
paddle_b.goto(150, 0)

#Loptica
ball = Turtle()
ball.speed(0)
ball.shape("circle")
ball.color("white")
ball.penup()
ball.goto(0, 0)
ball.dx = 1
ball.dy = -1

#Pen-napis
pen = Turtle()
pen.speed(0)
pen.color("white")
pen.penup()
pen.hideturtle()
pen.goto(0, 110)
pen.write("Igrač A: 0          Igrač B: 0", align="center", font=("Arial", 15, "normal"))

wind.listen()
wind.onkeypress(paddle_a_up, "w")
wind.onkeypress(paddle_a_down, "s")

#Kreiranje socketa
s=socket(AF_INET, SOCK_STREAM)
#IP servera
host = '127.0.0.1'
#Port
port = 7010
#Povezivanje IP i porta jedno s drugim
s.bind((host, port))
#Čekamo da se klijent poveže
s.listen(1)
print ("Uspostavljen poslužitelj...")

#Prihvaćen zahtjev za povezivanje od klijenta
#vraćamo broj sesije i IP / port klijenta
c, add=s.accept()
print ("Uspostavljena veza...")
#Nit za posluživanje sesije
start_new_thread(primi_thread,(c,))

start_new_thread(igra,(c,))
wind.mainloop()


import serial
import numpy as np
import csv

device=serial.Serial("/dev/ttyUSB0",230400)
distance0=1
distance1=7.5

##mm
arange=np.arange(distance0,distance1,0.1)
f= open("data/trail9-5.4mm.csv","w")
writer=csv.writer(f)
for i in arange:
    print("Move Slit to: ",str(i))

    input("Press enter to continue")

    for a in range(10):
        device.reset_input_buffer()
        line=device.readline()
        nice= line.strip((b'\n')).split(b',')
        print(a)
        nice.append(i)
        writer.writerow(nice)

f.close


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# PYTHON_ARGCOMPLETE_OK

"""
SVG Templating System (C) Max Gaukler and other members of the FAU FabLab 2013-2022
unlimited usage allowed, see LICENSE file
"""

from lxml import etree
from copy import deepcopy
import inspect
from io import BytesIO
import re
import locale
import codecs
import argparse
import argcomplete
import sys
import os
import subprocess
from json import loads, dumps
from repoze.lru import lru_cache  # caching decorator for time-intensive read functions
from logging import error, warning
import requests

__author__ = 'Max Gaukler, sedrubal'
__license__ = 'unlicense'

# <editor-fold desc="argparse">
parser = argparse.ArgumentParser(description='Automated generating of labels for products from the ERP-web-API')
parser.add_argument('ids', metavar='ids', type=str, nargs='*', default='',
                    help='the ids of the products (4 digits) or purchase orders (PO + 5 digits) to generate a label. '
                         'You can use expressions like 5x1337 to print 5 times the label for 1337. '
                         'And you can also use stdin for ids input. '
                         'Can\'t be used with \'json-input\'.')
parser.add_argument('-o', '--json-output', action='store_true', dest='json_output',
                    help='use this, if you only want to fetch the data for the labels from the ERP-web-API '
                         'and if you want to read the data as json from stdout. '
                         'Can\'t be used with \'json-input\'.')
parser.add_argument('-i', '--json-input', action='store_true', dest='json_input',
                    help='use this, if the data for the labels should be provided through stdin as json '
                         'instead of fetching it from ERP-web-API. '
                         'Can\'t be used with \'ids\' and \'json-output\'.')

argcomplete.autocomplete(parser)
args = parser.parse_args()
# </editor-fold>

# <editor-fold desc="create svg label">
def clear_group_members(tree, group):
    """
    removes all content of a given group in a svg
    :param tree: the svg tree
    :param group: name of the group
    """
    for e in tree.findall(".//{http://www.w3.org/2000/svg}g[@id='" + group + "']/*"):
        e.clear()

# <editor-fold desc="barcode">
def make_barcode_xml_elements(string, barcode):
    """
    generates an EAN8 barcode and returns a lst of lxml-elements
    :param string: text to be encoded as a barcode
    :param barcode: (?)
    :return: a list of lxml elements
    """
    # Pseudo-Datei-Objekt
    s = BytesIO()
    ean8 = barcode.get_barcode_class('ean8')
    b = ean8(string)
    b.write(s)
    # oder zu Debugzwecken: barcode.save('barcode') speichert in barcode.svg
    barcode_elements = etree.fromstring(s.getvalue())
    s.close()
    return barcode_elements.findall(".//{http://www.w3.org/2000/svg}rect")

def ean8_check_digit(num):
    """
    EAN checksum

    gewichtete Summe: die letzte Stelle (vor der Pruefziffer) mal 3,
    die vorletzte mal 1, ..., addiert.

    Pruefziffer ist dann die Differenz
    dieser Summe zum naechsten Vielfachen von 10

    :param num: number to be encoded as EAN
    :return: checksum digit
    """
    s = str(num)[::-1]  # in string wandeln, umkehren
    checksum = 0
    even = True
    for char in s:
        n = int(char)
        if even:
            n *= 3
        checksum += n
        even = not even
    return (10 - (checksum % 10)) % 10

def create_ean8(num):
    """
    baue gueltige EAN8 aus Zahl:

    - vorne Nullen auffuellen
    - wenn Zahl kleiner 10000, mache eine EAN8 im privaten Bereich daraus: 200nnnn
    - add checksum digit

    :param num: number for the barcode
    :return: (?)
    """
    if len(str(num)) == 8:
        return str(num)
    num = int(num)
    if num < 10000:
        num += 2000000
    return '%07d%d' % (num, ean8_check_digit(num))
# </editor-fold>

def make_label(data, etikett_num, barcode, label_template):  # , dict_input
    """
    Generates a label with following information
    :param data: a dict containing the data for the label
    :param etikett_num: the number (?) of the label
    :param barcode: the barcode (generated from the product ID (?))
    :param label_template: the template for labels
    :return: a label in svg (?) or None, when the product couldn't be found
    """
    # :param dict_input: deprecated
    etikett = deepcopy(label_template)
    etikett.set("id", "etikettGeneriert" + str(etikett_num))

    if len(data) == 0:
        return None

    # replace all text
    replacements = [[key, value] for [key, value] in data.items() if key in ['ID', 'ORT', 'PREIS', 'TITEL', 'VERKAUFSEINHEIT']]
    for element in etikett.iter("*"):
        for [key, value] in replacements:
            if element.tail is not None and key in element.tail:
                element.tail = element.tail.replace(key, value)
                break # break to avoid double substitution (e.g. "TITEL" -> "3D-Druck NORMALPREIS" -> "3D-Druck Normal0,30 €")
        for [key, value] in replacements:
            if element.text is not None and key in element.text:
                element.text = element.text.replace(key, value)
                break  # break to avoid double substitution (e.g. "TITEL" -> "3D-Druck NORMALPREIS" -> "3D-Druck Normal0,30 €")
    for e in make_barcode_xml_elements(create_ean8(data["ID"]), barcode):
        etikett.find(".//{http://www.w3.org/2000/svg}g[@id='barcode']").append(e)
    etikett.find(".//{http://www.w3.org/2000/svg}g[@id='barcode']").set("id", "barcode" + str(etikett_num))
    return etikett
# </editor-fold>

@lru_cache(1)
def read_product_db():
    r = requests.get('https://brain.fablab.fau.de/build/pricelist/price_list-Alle_Produkte.html.json')
    return r.json()

# <editor-fold desc="fetch data">
# <editor-fold desc="fetch data from oerp">
@lru_cache(1024)
def read_product(product_id):
    """
    Fetches the data for the requested product
    :param product_id: the openERP product ID of the requested product
    :param oerp: the openERP lib instance
    :return: a data dict or an empty dict if the product couldn't be found
    """
    # produktRef='0009'
    # adds leading 0

    product_id = int(product_id)
    product_id_zeroes = "{:04}".format(product_id)
    products = read_product_db()

    if product_id_zeroes not in products:
        error("ID %d nicht gefunden!" % product_id)
        return {}
    p = products[product_id_zeroes]
    location_string = p["_location_str"]
    verkaufseinheit = p['_per_uom_str']
    price = p['_price_str']

    data = {"TITEL": p['name'], "ORT": location_string, "ID": product_id_zeroes,
            "PREIS": price,
            "VERKAUFSEINHEIT": verkaufseinheit}

    return data

@lru_cache(128)
def get_ids_from_order(po_id):
    """
    Fetches the product IDs of a purchase order
    :param po_id: The openERP purchase order ID
    :param oerp: the openERP lib instance
    :return: an array containing the openERP product IDs of a purchase
    """
    error("purchase orders (PO1234) are currently not supported. We first need to create a JSON exporter for that to have an API that works with Py3")
    # return [1, 42, 2937]
# </editor-fold>

def read_products_from_stdin():
    """
    Reads the json label description from stdin
    :return: a dict containing the information for the labels
    """
    labels_data = loads(read_stdin())
    return labels_data
# </editor-fold>

def read_stdin():
    """
    Reads text from stdin
    :return: the text given through stdin
    """
    text = sys.stdin.read()
    return text

def main():
    """
    The main class of the script: generate labels for products and produces a pdf file
    :raise Exception:
    """
    # <editor-fold desc="check arguments">
    if args.json_input and args.json_output:
        error("Invalid arguments. If you don't want to create a PDF-label you can't provide data through stdin.")
        parser.print_help()
        exit(1)
    elif args.json_input and args.ids:
        error("Invalid arguments. If you want to use the stdin for json data input, you mustn't provide ids.")
        parser.print_help()
        exit(1)
    # </editor-fold>

    script_path = os.path.realpath(os.path.dirname(inspect.getfile(inspect.currentframe())))  # path of this script

    if not args.json_input:
        # <editor-fold desc="evaluate input, replace PO IDs with their product ids, fetch data from oerp">
        purchase_regex = re.compile(r"^(\d{1,2}x)?po\d{1,5}$")  # (a number and 'x' and) 'PO' or 'po' and 1-5 digits
        product_regex = re.compile(r"^(\d{1,2}x)?\d{1,4}$")  # (a number and 'x' and) 1 to 4 digits
        labels_data = dict()
        if len(args.ids):
            input_ids = args.ids
        else:
            input_ids = read_stdin().strip().split(' ')
        for args_id in input_ids:
            number_of_labels = 1
            args_id = args_id.lower()
            if 'x' in args_id:
                number_of_labels_str = args_id[:3].split('x', 2)[0]
                assert number_of_labels_str.isdigit(), "invalid input"
                # multiple labels requested: (1-25)x(product_id)
                number_of_labels = max(0, min(int(number_of_labels_str), 25))
                x_position = args_id.find('x')
                args_id = args_id[x_position + 1:]
            if purchase_regex.match(args_id):
                prod_ids = get_ids_from_order(args_id)
                for prod_id in prod_ids:
                    prod_id = int(prod_id)
                    if prod_id not in labels_data.keys():
                        prod_data = deepcopy(read_product(prod_id))
                        if len(prod_data):
                            labels_data[prod_id] = prod_data
                            labels_data[prod_id]['COUNT'] = number_of_labels
                    else:
                        labels_data[prod_id]['COUNT'] += number_of_labels
            elif product_regex.match(args_id):
                args_id = int(args_id)
                if args_id not in labels_data.keys():
                    prod_data = deepcopy(read_product(args_id))
                    if len(prod_data):
                        labels_data[args_id] = prod_data
                        labels_data[args_id]['COUNT'] = number_of_labels
                else:
                    labels_data[args_id]['COUNT'] += number_of_labels
            else:
                error("The ID '" + args_id + "' you entered is invalid.")
                exit(1)
            if not labels_data:
                error("No valid products found. Products must have a valid 'internal ID' like 0123.")
                exit(1)
        # </editor-fold>
    else:
        labels_data = read_products_from_stdin()
        label_count = 0
        for prod in labels_data.values():
            label_count += prod['COUNT']
        if label_count > 50:
            error("Too many labels!")
            exit(1)

    if args.json_output:
        print(dumps(labels_data, sort_keys=True, indent=4, separators=(',', ': ')))  # json.dumps in pretty
    else:
        import barcode

        # <editor-fold desc="load page template (for labels) and empty is">
        template = etree.parse(script_path + "/vorlage-etikettenpapier-60x30.svg")

        # remove everything with an id starting with 'ignore'
        for e in template.findall("*"):
            if e.get("id", "").endswith("ignore"):
                e.clear()
        # pick out items
        # they need to be directly on the root level in the file
        # (or at least not inside a group with an applied transformation), so that position and size is correct
        etikett_template = deepcopy(template.find(".//{http://www.w3.org/2000/svg}g[@id='etikett']"))
        clear_group_members(etikett_template, 'barcode')
        clear_group_members(template, 'etikett')
        # </editor-fold>

        # <editor-fold desc="deprecated things">
        # <editor-fold desc="tab-newline-separated data aus google doc">
        # url=urllib2.urlopen("https://docs.google.com/spreadsheet/pub?key=0AlfhdBG4Ni7BdFJtU2dGRDh2MFBfWHVoUEk5UlhLV3c&single=true&gid=0&output=txt")
        # textInput=url.read().decode('utf-8')
        # # convert to array
        # listInput=[]
        # for line in textInput.split('\n'):
        # listInput.append(line.split('\t'))
        # # HARDCODED: the fourth column contains the column name
        # columnNames=listInput[3]
        # # convert to dictionary: {"SPALTENNAME":"Inhalt",...}
        # </editor-fold>
        # dict_input = {}
        # for line in listInput:
        # n=0
        # d={}
        # for col in line:
        # d[columnNames[n]]=col
        # n=n+1
        # dict_input[d["ID"]]=d
        # print(p)
        # p['lst_price'] p['name'] p['description']

        # Fehler vermeiden: wenn leere Ausgabe gefordert, erzeuge eine leere Seite, statt garnix
        # if len(product_ids) == 0:
        # product_ids = [None]
        # </editor-fold>

        # <editor-fold desc="make temp dir">
        output_dir = script_path + '/public/temp/'
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        # </editor-fold>

        page_count = 0
        pdfs_to_merge = []

        for label_data in labels_data.values():
            # <editor-fold desc="generate and save a svg->pdf for each label"
            page = deepcopy(template)
            for i in range(label_data['COUNT']):
                label_svg = make_label(label_data, 0, barcode, etikett_template)
                if label_svg is not None:
                    page.getroot().append(label_svg)
                    # <editor-fold desc="write svg and convert it to pdf">
                    output_file_base_name = "output-etikettenpapier-%d" % page_count
                    svg_file = output_dir + output_file_base_name + ".svg"
                    pdf_file = output_dir + output_file_base_name + ".pdf"
                    page.write(svg_file)
                    subprocess.call("inkscape {in_file} --export-filename={out_file} 2>&1 | egrep -v '(^$|dbus|Failed to get connection)'".format(
                        in_file=svg_file,
                        out_file=pdf_file
                    ), shell=True)
                    # </editor-fold>
                    pdfs_to_merge.append(pdf_file)
                    page_count += 1
            # <editor-fold>

        # <editor-fold desc="append pages (pdftk)"

        pdf_output = output_dir + "output-etikettenpapier.pdf"
        subprocess.check_call(["qpdf", "--empty", "--pages"]  + pdfs_to_merge + ["--", pdf_output])
        # </editor-fold>
        # <editor-fold desc="clean">
        for p in range(page_count):
            try:
                os.remove(output_dir + ("output-etikettenpapier-%d.pdf" % p))
            except OSError:
                    pass
            try:
                os.remove(output_dir + ("output-etikettenpapier-%d.svg" % p))
            except OSError:
                pass
        # </editor-fold>

    exit(0)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# Create status info of ISC DHCPd leases in JSON format.
#
# Depends on python module netaddr.
#   Install with pip install netaddr.
# Depends on iscconf for parsing dhcpd.conf.
#   Install with pip install iscconf
# Depends on python 2.7 for argparse.
#
# This was written because dhcpstatus kept crashing with OOM errors on a
# 500M large leases file.
#
# Run dhcpstatus.py -h for more info.
#
# By Stefan Midjich

from __future__ import print_function

from sys import exit, stderr, stdout
from argparse import ArgumentParser, FileType
from json import dumps

from netaddr import IPNetwork
from iscconf import parse

# This does the actual counting of leases in dhcpd.leases. It takes a list of
# valid IP-addresses as argument, and a file object to the leases.
# Return value is a dictionary of matched leases.
# Counting can take a long time on big leases files.
def count_leases(file, valid_ips):
    matched_ips = {}
    for line in file:
        if line.startswith('lease '):
            (junk, current_ip, junk) = line.split(' ')
            if current_ip in valid_ips:
                try:
                    matched_ips[current_ip]['count'] += 1
                except:
                    matched_ips[current_ip] = {
                        'count': 1,
                        'starts': None,
                        'ends': None
                    }
        else:
            if line.lstrip(' ').startswith('starts '):
                (
                    junk1,
                    junk2,
                    date,
                    time
                ) = line.lstrip(' ').split(' ')
                try:
                    matched_ips[current_ip]['starts'] = date
                except:
                    pass
            if line.lstrip(' ').startswith('ends '):
                (
                    junk1,
                    junk2,
                    date,
                    time
                ) = line.lstrip(' ').split(' ')
                try:
                    matched_ips[current_ip]['ends'] = date
                except:
                    pass
    return matched_ips

def get_valid_ips(subnets, ips):
    for subnet in subnets:
        _ip = IPNetwork(subnet)
        for _ip_address in list(_ip):
            ips.append(str(_ip_address))

arse = ArgumentParser(
    description = 'Create JSON statistics of used leases in ISC DHCPd',
    epilog = '''This program works by reading all the shared-network blocks in
    a dhcpd.conf file. Only subnets in shared-network blocks are processed.

    By Stefan Midjich'''
)

arse.add_argument(
    '-v', '--verbose',
    action = 'store_true',
    help = 'Debugging info'
)

arse.add_argument(
    '-l', '--leases',
    metavar = '/var/lib/dhcp/dhcpd.leases',
    default = '/var/lib/dhcp/dhcpd.leases',
    type = FileType('r'),
    help = 'File containing all leases for ISC DHCPd'
)

arse.add_argument(
    '-L', '--list',
    action = 'store_true',
    help = 'Only list available subnets and exit'
)

arse.add_argument(
    '-i', '--indent',
    action = 'store_true',
    help = 'Indent output to prettify it'
)

arse.add_argument(
    '-c', '--configuration',
    metavar = '/etc/dhcp/dhcpd.conf',
    default = '/etc/dhcp/dhcpd.conf',
    type = FileType('r'),
    help = 'ISC DHCPd Configuration file containing shared-network blocks'
)

arse.add_argument(
    '-o', '--output',
    metavar = '/tmp/dhcpstatus_output.json',
    default = stdout,
    type = FileType('w'),
    help = 'JSON output file for DHCP statistics'
)

arse.add_argument(
    'isp_name',
    metavar = 'ISP-name',
    help = 'Name of shared-network to create statistics for, can be \'any\''
)

args = arse.parse_args()

indent = None
if args.indent:
    indent = 4

try:
    parsed_dhcp_config = parse(args.configuration.read())
except Exception as e:
    print(str(e), file=stderr)
    arse.print_usage()
    exit(1)

# Start building the output dict by reading subnet info for each ISP defined
# in dhcpd.conf as a shared-network.
json_isp = {}
for item in parsed_dhcp_config:
    try:
        (k, v) = item
    except:
        continue
    if k == 'shared-network':
        last_isp = v
        json_isp[last_isp] = {}
        for subitem in parsed_dhcp_config[item]:
            try:
                (sk, sv, sk2, sv2) = subitem
            except:
                continue
            if sk == 'subnet':
                _subnet = sv + '/' + sv2
                try:
                    json_isp[last_isp]['subnets'].append(_subnet)
                except:
                    json_isp[last_isp]['subnets'] = [_subnet]

# Just list the ISPs and their subnets, and exit.
if args.list:
    if args.isp_name == 'any':
        print(dumps(json_isp, indent=indent), file=args.output)
    else:
        print(dumps(json_isp[args.isp_name], indent=indent), file=args.output)
    exit(0)

# Else proceed with regular execution of the program.

# Get a list of valid IP-addresses to search the leases for.
search_ips = []
if args.isp_name == 'any':
    for isp in json_isp:
        get_valid_ips(json_isp[isp]['subnets'], search_ips)
else:
    get_valid_ips(json_isp[args.isp_name]['subnets'], search_ips)

matched_ips = count_leases(args.leases, search_ips)

print(dumps(matched_ips, indent=indent), file=args.output)


#!/usr/bin/env python3.6
"""Simulate trees under site + gene + lineage + site x lineage variation."""

import variation
import my_module as mod

def main():
    """Simulate gene trees for genes with gene + lineage + sitexlineage variation.

    This will involve a base tree for each bin per gene
    each gene will have these trees for each partition multiplied by
    the gene rate"""

    base_tree = mod.get_file_data("../timetree.nwk")[0]
    params = variation.Settings()

    out = open("treefiles.nwk", "w")
    relaxed_trees = []
    i = 0
    while i < 70:
        j = 0
        rate = variation.get_rate(params.gene_alpha, params.gene_beta)
        while j < params.n_bins_per_gene:
            relaxed_trees.append(variation.relaxed_tree(base_tree,
                                                        rate,
                                                        params.lineage_sigma2
                                                        ))
            j += 1
        out.write("\t".join(relaxed_trees) +"\n")
        i += 1
    out.close()

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
import time

MAX_EPISODES = 13   # 游戏回合
FRESH_TIME = 0.3    # 每步移动的时长
ALPHA = 0.1         # 学习速率
GAMMA = 0.99        # 折减系数

class Maze_Env():
    def __init__(self):
        self.N_STATES = 6   # 一维迷宫的长度
        self.ACTIONS = ['left', 'right']    # agient 拥有的操作

    def reset(self):
        '''初始化迷宫,并返回初始状态'''
        state = 0
        print('\r{}'.format(''.join(['o'] + (['-']*(self.N_STATES-2)) + ['T'])), end='')
        return state

    def step(self, state, action):
        '''agent 与环境交互'''
        next_state, reward, terminated = None, 0, False

        if action == 'right':   # 向右移动
            if state == self.N_STATES - 2:   # 找到宝藏
                terminated = True
                reward = 1
            else:
                next_state = state + 1
        else:   # 向左移动
            if state == 0:
                next_state = state  # 撞到墙
            else:
                next_state = state - 1

        if not terminated:
            self.update_env(next_state)

        return next_state, reward, terminated

    def update_env(self, next_state):
        '''更新迷宫
        '''
        env_list = ['-']*(self.N_STATES-1) + ['T']
        env_list[next_state] = 'o'
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)

    def get_n_states(self):
        return self.N_STATES

    def get_actions(self):
        return self.ACTIONS

class Agent():
    def __init__(self, env):
        self.EPSILON = 0.9   # greedy police
        self.n_states = env.get_n_states()
        self.actions = env.get_actions()
        self.q_table = self.build_q_table()

    def build_q_table(self):
        table = pd.DataFrame(
            # q_table 的初始值
            np.zeros((self.n_states, len(self.actions))),
            columns=self.actions,    # q_table 列名为可执行动作名称
        )
        return table

    def choose_action(self, state):
        '''选择动作
        '''
        state_actions = self.q_table.iloc[state, :]  # 得到当前状态的可选择动作
        if (np.random.uniform() > self.EPSILON) or ((state_actions == 0).all()):    # 非随机模式 或者 当前的状态下的Q值无效
            action_name = np.random.choice(self.actions)
        else:   # act greedy
            action_name = state_actions.idxmax()    # 选择对应Q值最大的动作
        return action_name

    def update_q_table(self, state, action, reward, done, next_state):
        q_predict = self.q_table.loc[state, action]

        if done:
            q_target = reward     # 下一个状态是终端
        else:
            q_target = reward + GAMMA * self.q_table.iloc[next_state, :].max()   # 下一个状态不是终端

        self.q_table.loc[state, action] += ALPHA * (q_target - q_predict)  # 更新

if __name__ == "__main__":
    env = Maze_Env()
    agent = Agent(env)

    for episode in range(MAX_EPISODES):
        step_counter = 0
        state = env.reset()  # 初始化环境

        while True:

            action = agent.choose_action(state)

            next_state, reward, done = env.step(state, action)  # 采取行动并获得下一个状态和奖励

            agent.update_q_table(state, action, reward, done, next_state)

            state = next_state  # 移至下一个状态

            step_counter += 1

            if done:
                interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
                print('\r{}'.format(interaction), end='')
                time.sleep(2)
                print('\r                                ', end='')
                break

    print('\r\nQ-table:\n')
    print(agent.q_table)


"""
@Filename:  core/testcase/test_login.py
@Author:    小蔡
@Time:      2022/09/28 20:30
@Describe: ...
"""
import json

import requests

# def img2code(file):
#     url = "http://47.106.70.21:5516/auth/login"
#
#     data = {
#         "user": "pengjiangchun",
#         "pass2": "3eb5f9a3a31fb000969e696d7c3cc71f",  # 密码的md5值
#         "sofid": "mashang",  # 个人中心设置的软件id
#         "codetype": 1902,
#     }
#     files = {"userfile": open(file, "rb")}
#     resp = requests.post(url, data=data, files=files)
#
#     res = resp.json()
#     if res["err_no"] == 0:
#         code = res["pic_str"]
#         print("识别成功", code)
#         return code
#     else:
#         print("识别失败")
#         return False

def save_cookies(driver):
    """保存cookies到本地文件"""
    cookies = driver.get_cookies()  # 获取cookies

    with open("temp/cookies.json", "w") as f:
        f.write(json.dumps(cookies))

def load_cookies(driver):
    """从本地文件加载cookies"""

    driver.get("https://scrm-uat.immotors.com/marketing/dataBoard/page?_tgt=fr")
    try:
        with open("temp/cookies.json") as f:
            cookies = json.loads(f.read())

        for cookie in cookies:
            driver.add_cookie(cookie)  # 使用cookies

        driver.refresh()  # 刷新页面，向服务器发送cookeis
    except:
        pass

def is_login(driver):
    """是否已经登录"""
    if "请输入密码" in driver.page_source:
        print("需要登录")
        return False
    else:
        print("已经登录"+'\n')
        return True


import sys

from hypothesis.version import __version__

message = """
Hypothesis {} requires Python 3.6 or later.

This can only happen if your packaging toolchain is older than python_requires.
See https://packaging.python.org/guides/distributing-packages-using-setuptools/
"""

if sys.version_info[:3] < (3, 6):  # pragma: no cover
    raise Exception(message.format(__version__))


import pandas as pd
from sodapy import Socrata
from math import ceil

soil_variables = ['ph_agua_suelo_2_5_1_0', 'potasio_k_intercambiable_cmol_kg', 'f_sforo_p_bray_ii_mg_kg']

def create_client():
    client = Socrata("www.datos.gov.co", None)
    return client

def get_data(dataset_identifier, **kwargs):
    client = create_client()
    result = client.get(dataset_identifier, **kwargs)
    return result

def convert_dataset_to_df(data):
    result_df = pd.DataFrame.from_records(data)
    if result_df.shape[1] == 0:
        raise ValueError(
            "No se encontraron valores con estos parametros, verifique que haya escrito todo de manera correcta")
    return result_df

def data_normalize(dataset_values):
    to_float = lambda x: float(x)

    for iterable in range(len(dataset_values)):
        try:
            dataset_values[iterable] = to_float(dataset_values[iterable])
        except ValueError:
            dataset_values.pop(iterable)

def calculate_median(data):
    medians = {}
    for soil_variable in soil_variables:
        values = data[soil_variable]
        data_normalize(values)
        length = len(values)

        if length % 2 == 0:
            poss_median1 = ceil(length / 2)
            poss_median2 = poss_median1 - 1
            median = (values[poss_median1] + values[poss_median2]) / 2
            medians[soil_variable] = median

        else:
            poss_median = ceil(length / 2)
            medians[soil_variable] = values[poss_median]

    return medians

def get_relevant_info(data):
    columns_dataframe = ["departamento", "municipio", "cultivo", "topografia"] + soil_variables
    new_df = pd.DataFrame()

    for column_name in columns_dataframe:
        new_df[column_name] = data[column_name]
    return new_df

def normalize_params(params):
    params["departamento"] = params["departamento"].upper()
    params["municipio"] = params["municipio"].upper()
    params["cultivo"] = params["cultivo"].title()


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 24 09:27:21 2017

@author: Admin
"""

import pandas as pd

### READ IN DATA SOURCE
### READ DIFFRENT SECTIONS FOR TRAIN, TEST, PREDICT
df_train = pd.read_csv('LMPD_STOPS_DATA_CLEAN_V1_HEADERS.csv', nrows=100000, skipinitialspace=True)
# SAve Headers for the next selections that skip header row.
headers = list(df_train)

df_test = pd.read_csv('LMPD_STOPS_DATA_CLEAN_V1_HEADERS.csv',names=headers,skiprows=100001, nrows=1000, skipinitialspace=True)
#df_predict = pd.read_csv('LMPD_STOPS_DATA_CLEAN_V1_HEADERS.csv',names=headers,skiprows=93001, nrows=100, skipinitialspace=True)

df_train =df_train.dropna(axis=0, how='any')
df_test =df_test.dropna(axis=0, how='any')
#df_predict =df_predict.dropna(axis=0, how='any')

obj_columns = df_train.select_dtypes(['object']).columns
df_train[obj_columns] = df_train[obj_columns].apply(lambda x: x.astype('category'))
df_test[obj_columns] = df_test[obj_columns].apply(lambda x: x.astype('category'))
#df_predict[obj_columns] = df_predict[obj_columns].apply(lambda x: x.astype('category'))

cat_columns = df_train.select_dtypes(['category']).columns
df_train[cat_columns] = df_train[cat_columns].apply(lambda x: x.cat.codes)
df_test[cat_columns] = df_test[cat_columns].apply(lambda x: x.cat.codes)
#df_predict[cat_columns] = df_predict[cat_columns].apply(lambda x: x.cat.codes)
#####################################################

########################
df_train_data = df_train.drop('ACTIVITY_RESULTS', 1)
train_data =  df_train_data.values
train_target = df_train.ACTIVITY_RESULTS.values
##############
df_test_data = df_test.drop('ACTIVITY_RESULTS', 1)
test_data =  df_test_data.values
test_target = df_test.ACTIVITY_RESULTS.values
############################

# Create and fit a nearest-neighbor classifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn import linear_model
from sklearn import svm
model = linear_model.LinearRegression()

model.fit(train_data, train_target)

predict = model.predict(test_data)

df_output = pd.DataFrame(
    {'Prediciton': predict,
     'Target': test_target
    })

df_output.Prediciton = df_output.Prediciton.round()

df_output['Match'] = df_output['Prediciton'] == df_output['Target']

accuracy = df_output.Match.value_counts(normalize=True)

print (accuracy)