# implementing pymoo multi-objective optimization with FEniCS FEA objectives

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.offsetbox import TextArea, DrawingArea, OffsetImage, AnnotationBbox
import pathlib
from pathlib import Path

from pymoo.core.problem import ElementwiseProblem
from pymoo.algorithms.moo.nsga2 import NSGA2
from pymoo.termination import get_termination
from pymoo.optimize import minimize

from pymoo.operators.sampling.rnd import BinaryRandomSampling
from pymoo.operators.crossover.hux import HalfUniformCrossover
from pymoo.operators.crossover.ux import UniformCrossover
from pymoo.operators.crossover.pntx import TwoPointCrossover
from pymoo.operators.mutation.bitflip import BitflipMutation
from pymoo.decomposition.asf import ASF

# importing required modules (have to be in the same directory!)
from core_functions import *
from boundary_conditions import *

# defining the multiobjective problem for square FEA problem: Weight/vm_stress obj
class MyProblem(ElementwiseProblem):

    def __init__(self, conn, coords, num_el, meshpath):
        super().__init__(n_var=num_el,
                         n_obj=2,
                         n_ieq_constr=0,
                         xl=np.zeros(num_el),
                         xu=np.ones(num_el)
                         )
        self.conn = conn
        self.coords = coords
        self.num_el = num_el
        self.meshpath = meshpath  # temp mesh for evaluation

    def _evaluate(self, x, out, *args, **kwargs):

        # generate the infill file
        generate_infill_file(self.meshpath, x, self.conn, self.coords)

        # Loading the created infill mesh to FEniCS
        mesh, _, _, _ = extract_mesh_data_fenics(meshfile)

        # perform FEA and obtain displacements and von mises stress
        d_max, d_tot, d_avg, max_vm = square_FEA(mesh)

        # objective values: we want to minimize *weight* and *maximal stress*
        f1 = np.sum(x)  # objective 1 - "weight"
        f2 = d_max     # objective 2 - "maxStress"

        # constraints
        #g1 = np.sum(x) - 200   # dummy constraint: can be adjusted to limit the infill rate

        out["F"] = [f1, f2]  # dictionary key for objectives
        #out["G"] = [g1]      # dictionary key for constraints

# Filepath management
cwd = Path.cwd().parent
mesh_dir = cwd.joinpath("MOEA_meshes", "smallSquare7x7")  # adjust the dir here
inf_file = str(mesh_dir.stem) + '-infill.msh'
mesh_inp = mesh_dir / inf_file  # mesh input file

# Extracting geometry from the infill mesh -> num_el is the design dimension
conn, coords, num_el = extract_mesh_data(mesh_inp)   # extracting the geometry

# init the MOP
meshfile = mesh_dir / 'infill_gen.msh'
problem = MyProblem(conn, coords, num_el, meshfile)

# initialize the algorithm object
algorithm = NSGA2(
    pop_size=100,
    n_offsprings=40,
    sampling=BinaryRandomSampling(),
    crossover=TwoPointCrossover(prob=1.0),
    #crossover=UniformCrossover(),
    mutation=BitflipMutation(prob=0.5, prob_var= 2 * 1/num_el),  # mutation scaled by the num of elements
    eliminate_duplicates=True
)

# define the termination criterion
termination = get_termination("n_gen", 10)

# solve the MOP
res = minimize(problem,
               algorithm,
               termination,
               seed=1,
               save_history=True,
               verbose=True)

# n_gen  - generation counter,
# n_eval - number of evaluations
# cv_min - minimum constraint violation
# cv_avg - average constraint violation in the current population.
# n_nds  - number of non-dominated solutions
# eps/ind- running performance metrics

X = res.X  # solutions in the decision space
F = res.F  # solutions in the objective space
hist = res.history  # getting the info for each iteration

#print(X)
#print(F)

# save solutions
out_dir = mesh_dir / "infill_gen"
img_dir = out_dir / "png"
out_dir.mkdir(exist_ok=True)
img_dir.mkdir(exist_ok=True)
for i, x in enumerate(X):
    ifile = out_dir / f"sol{i:02d}.msh"  # generate mesh
    generate_infill_file(ifile, x, conn, coords)
    imfile = img_dir / f"sol_{i}.png"    # generate png
    generate_img_file(imfile, x, conn, coords)
    print(f"File {ifile.name} saved.")###

# writing to a file
with open(out_dir / "sq_solutions.txt", "w+") as file:
    alg = str(type(algorithm))[8:-2]
    header = f"Algorithm:{alg}, pop_size: {algorithm.pop_size},\
     n_offsp: {algorithm.n_offsprings}, n_gen: {termination.n_max_gen},\
     t_tot: {res.exec_time:10.1f}, t_gen: {res.exec_time/termination.n_max_gen:10.2f}\n"
    file.writelines(header)
    file.writelines(f"{str(list(sol))} \t w: {F[i][0]:10.1f} \t vm: {F[i][1]:10.06f} \n" for i, sol in enumerate(1*X))

# plotting the decision space
xl, xu = problem.bounds()
plt.figure(figsize=(7, 5))
plt.imshow(X)
plt.title("Design Space")
plt.show()

# plotting the objective space
plt.figure(figsize=(5, 3))
plt.scatter(F[:, 0], F[:, 1], s=20, facecolors='black', edgecolors='black')
plt.title("Objective Space")
plt.savefig(out_dir / 'ObjectiveSpace.png')
plt.show()

# Multi-Criteria Decision Making: subset selection
n_keep = 9  # how many solutions should we keep
if len(X) > n_keep:
    approx_ideal = F.min(axis=0)  # ideal point
    approx_nadir = F.max(axis=0)  # nadir point

    # normalizing with respect to both objectives
    nF = (F - approx_ideal) / (approx_nadir - approx_ideal)  # normalized objectives

    # Decision making: Using Augmented Scalarization Function (ASF)
    decomp = ASF()                  # init the ASF metric to be minimized
    weights = [np.array([(k + 1)/(n_keep+1), (n_keep - k)/(n_keep+1)]) for k in range(n_keep)]
    best_ASF = []  # indexes of chosen solutions
    for weight in weights:
        best_ASF.append(decomp.do(nF, 1/weight).argmin())  # index of the best solution regarding ASF
    best_ASF = list(set(best_ASF))  # remove duplicates
    F_ASF = F[best_ASF, :]          # objectives
    X_ASF = X[best_ASF, :]          # solutions
    n_kept = len(best_ASF)          # number of kept solutions

    # plotting the objective space with mesh png annotations
    fig, ax = plt.subplots(figsize=(13, 8))
    plt.scatter(F_ASF[:, 0], F_ASF[:, 1], s=20, facecolors='black', edgecolors='black')

    for i, (x, y) in enumerate(F_ASF):
        ind = best_ASF[i]
        img_lab = plt.imread(img_dir / f"sol_{ind}.png")
        imagebox = OffsetImage(img_lab, zoom=0.045)
        imagebox.image.axes = ax

        fl_offset = (-1)**i * 20  # spreading fluctuating offset
        ab = AnnotationBbox(imagebox, (x, y),
                            xybox=(32 - 0.6 * fl_offset, 38 + fl_offset),
                            xycoords='data',
                            boxcoords=("offset points", "offset points"),
                            pad=-1,
                            arrowprops=dict(
                                arrowstyle="->",
                                connectionstyle="angle,angleA=0,angleB=90,rad=3",
                                linewidth=0.5)
                            )
        ax.add_artist(ab)
        ax.text(x, y, f"  {str(ind)}", va='top', ha='left', zorder=4)

    xrang = np.abs(np.max(F_ASF[:,0]) - np.min(F_ASF[:,0]))
    yrang = np.abs(np.max(F_ASF[:,1]) - np.min(F_ASF[:,1]))
    ax.set_xlim((np.min(F_ASF[:,0]) - 0.05*xrang, np.max(F_ASF[:,0]) + 0.15*xrang ))
    ax.set_ylim((np.min(F_ASF[:,1]) - 0.1*yrang, np.max(F_ASF[:,1]) + 0.25*yrang ))
    plt.title(f"Objective Space of {n_kept} ASF-selected solutions")
    plt.savefig(out_dir / 'ObjectiveSpace-ASF.png')
    plt.show()


'''Tests for the classification module.'''

import pytest
import torch
import torch.nn as nn
from torch.utils.data import TensorDataset, DataLoader
from sklearn.model_selection import train_test_split

from torchutils.classification import Classification

@pytest.fixture(params=[1, 10])
def data_num_features(request):
    return request.param

@pytest.fixture(params=[2, 10])
def data_num_classes(request):
    return request.param

@pytest.fixture(params=[1000])
def data_num_samples(request):
    return request.param

@pytest.fixture
def data_classification(data_num_features,
                        data_num_classes,
                        data_num_samples):
    '''Create classification problem.'''

    torch.manual_seed(0)

    num_features = data_num_features
    num_classes = data_num_classes
    num_samples = data_num_samples

    num_outputs = 1 if num_classes==2 else num_classes

    X = torch.randn(num_samples, num_features)
    y = torch.randint(num_classes, size=(num_samples,))

    X_train, X_test, y_train, y_test = train_test_split(
        X, y,
        test_size=0.2,
        random_state=0
    )

    train_set = TensorDataset(X_train, y_train)
    test_set = TensorDataset(X_test, y_test)

    train_loader = DataLoader(train_set, batch_size=32, shuffle=True)
    test_loader = DataLoader(test_set, batch_size=32, shuffle=False)

    model = nn.Linear(num_features, num_outputs)

    if num_outputs == 1:
        criterion = nn.BCEWithLogitsLoss(reduction='mean')
    else:
        criterion = nn.CrossEntropyLoss(reduction='mean')

    optimizer = torch.optim.Adam(params=model.parameters())

    classifier = Classification(
        model,
        criterion,
        optimizer,
        train_loader,
        test_loader,
        torch.device('cpu')
    )

    return classifier

def test_classifier_inference(data_classification):
    '''Test classifier at inference.'''

    classifier = data_classification
    classifier.train(False)

    X_batch, y_batch = next(iter(classifier.train_loader))
    X_batch = X_batch.to(classifier.device)
    y_batch = y_batch.to(classifier.device)

    with torch.no_grad():
        y_pred = classifier.predict(X_batch)
        y_proba = classifier.predict_proba(X_batch)
        y_topclass, _ = classifier.predict_top(X_batch)

    y_pred = y_pred.cpu()
    y_proba = y_proba.cpu()
    y_topclass = y_topclass.cpu()

    assert y_pred.shape == (X_batch.shape[0], classifier.model.out_features)
    assert y_proba.shape == (X_batch.shape[0], classifier.model.out_features)
    assert y_topclass.numel() == y_batch.numel()


import cv2 as cv
import numpy as np

left_window = 'Task 4 - left'
right_window = 'Task 4 - right'

# create window for left camera
cv.namedWindow(left_window)

# create window for right camera
cv.namedWindow(right_window)

# read in imagaes
img_l = cv.imread('imgs/combinedCalibrate/aL09.bmp')
img_r = cv.imread('imgs/combinedCalibrate/aR09.bmp')
gray = cv.cvtColor(img_l,cv.COLOR_BGR2GRAY)
shape = (gray.shape[1],gray.shape[0])

cv.imshow(left_window,img_l)
cv.imshow(right_window,img_r)
cv.waitKey(0)

# read in intrinsic params
fs_read_l = cv.FileStorage('params/left_cam.yml',cv.FILE_STORAGE_READ)
mat_l = fs_read_l.getNode("intrinsic").mat()
dist_l = fs_read_l.getNode("distortion").mat()
fs_read_l.release()

fs_read_r = cv.FileStorage('params/right_cam.yml',cv.FILE_STORAGE_READ)
mat_r = fs_read_r.getNode("intrinsic").mat()
dist_r = fs_read_r.getNode("distortion").mat()
fs_read_r.release()

fs_read = cv.FileStorage('params/stereo.yml',cv.FILE_STORAGE_READ)
R = fs_read.getNode('R').mat()
T = fs_read.getNode('T').mat()
fs_read.release()

# rectify images
R1,R2,P1,P2,Q,roi1,roi2 = cv.stereoRectify(mat_l,dist_l,mat_r,dist_r,shape,R,T)

map_l1, map_l2 = cv.initUndistortRectifyMap(mat_l,dist_l,R1,P1,shape,5)
map_r1, map_r2 = cv.initUndistortRectifyMap(mat_l,dist_l,R1,P1,shape,5)

rect_img_l = cv.remap(img_l,map_l1,map_l2,cv.INTER_LINEAR)
rect_img_r = cv.remap(img_r,map_r1,map_r2,cv.INTER_LINEAR)

diff_l = cv.absdiff(img_l,rect_img_l)
diff_r = cv.absdiff(img_r,rect_img_r)

def drawLines(img,color):
    x0 = -10
    x1 = 800
    for k in [1,2,3]:
        y = 100 * k
        image = cv.line(img,(x0,y),(x1,y),color,2)
    return image

rect_img_r = drawLines(rect_img_r,(0,0,255))
rect_img_l = drawLines(rect_img_l,(0,255,0))

cv.imshow(left_window,rect_img_l)
cv.imshow(right_window,rect_img_r)
cv.waitKey(0)

cv.imshow(left_window,diff_l)
cv.imshow(right_window,diff_r)
cv.waitKey(0)

cv.destroyAllWindows()
print('Program has ended')


from functools import partial

import chex
import jax
import numpy as np
import numpy.testing as npt
from absl.testing import parameterized

from tessellate_ipu import tile_map, tile_put_replicated
from tessellate_ipu.lax import scatter_add_p, scatter_max_p, scatter_mul_p, scatter_p

class IpuTilePrimitivesLaxScater(chex.TestCase, parameterized.TestCase):
    def setUp(self):
        super().setUp()
        self.device = jax.devices("ipu")[0]
        self.num_tiles = self.device.num_tiles
        # Not very clean, but for better reproducibility.
        np.random.seed(123)

    @parameterized.parameters(
        {"num_elements": 8, "num_indices": 3, "scatter_prim": scatter_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_add_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_max_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_mul_p},
        {"num_elements": 8, "num_indices": 3, "scatter_prim": scatter_add_p},
        {"num_elements": 8, "num_indices": 12, "scatter_prim": scatter_add_p},
        {"num_elements": 256, "num_indices": 512, "scatter_prim": scatter_add_p},
    )
    def test__tile_map__scatter__jitting__multi_sizes__proper_result(self, num_elements, num_indices, scatter_prim):
        tiles = (0,)
        data = np.random.randn(num_elements).astype(np.float32)
        indices = np.random.randint(low=0, high=num_elements, size=num_indices)
        indices = indices.reshape(-1, 1).astype(np.uint32)
        updates = np.random.randn(indices.size).astype(np.float32)

        # Only supported configuration!
        scatter_dnums = jax.lax.ScatterDimensionNumbers(
            update_window_dims=(), inserted_window_dims=(0,), scatter_dims_to_operand_dims=(0,)
        )

        def scatter_add_fn(data, indices, updates):
            data = tile_put_replicated(data, tiles)
            indices = tile_put_replicated(indices, tiles)
            updates = tile_put_replicated(updates, tiles)
            return tile_map(
                scatter_prim,
                data,
                indices,
                updates,
                dimension_numbers=scatter_dnums,
                indices_are_sorted=False,
                unique_indices=False,
                mode=jax.lax.GatherScatterMode.PROMISE_IN_BOUNDS,
                update_jaxpr=None,
                update_consts=None,
            )

        cpu_scatter_add_fn = partial(jax.jit, backend="cpu")(scatter_add_fn)
        ipu_scatter_add_fn = partial(jax.jit, backend="ipu")(scatter_add_fn)

        cpu_output = cpu_scatter_add_fn(data, indices, updates)
        ipu_output = ipu_scatter_add_fn(data, indices, updates)

        assert ipu_output.tiles == tiles
        assert ipu_output.dtype == data.dtype
        npt.assert_array_almost_equal(ipu_output, cpu_output)


import json
import os
import pickle
import numpy as np
from pprint import pprint

winningRuns, losingRuns, cards, relics, X, Y = None, None, None, None, None, None
cwd = os.path.dirname(__file__)
pathToData = os.path.join(cwd, '..', 'Data')

#Parses the wins and loses from the dump file
def loadFromFile(winsPath, losePath):
    #runsPath = os.path.join(pathToData, 'data_2018-10-24_0-5000.json')
    global winningRuns, losingRuns
    runs = []
    count = 0
    winningRuns = []
    losingRuns = []
    winningCount = 0
    lossCount = 0

    for f in os.listdir(os.path.join(pathToData, 'runs')):
        runsPath = os.path.join(pathToData, 'runs', f, f)
        runs = json.load(open(runsPath))
        count += len(runs)

        # If floors reached = 50 then died to final boss, 51 means victory
        for run in runs:
            floor = run['event']['floor_reached']
            if floor == 50:
                losingRuns.append(run)
                lossCount+=1
            elif floor == 51:
                winningRuns.append(run)
                winningCount += 1
        print("Reading: ", f, "Cumulative size: ", count, "NumWins: ", winningCount, "NumLosses", lossCount)

        if len(runs) > 200000:
            break

    #runs = json.load(open(runsPath))

    # global winningRuns, losingRuns
    # winningRuns = []
    # losingRuns = []
    #
    # #If floors reached = 50 then died to final boss, 51 means victory
    # for run in runs:
    #     floor = run['event']['floor_reached']
    #     if floor == 50:
    #         losingRuns.append(run)
    #     elif floor == 51:
    #         winningRuns.append(run)

    pickle.dump(winningRuns, open(winsPath, 'w'))
    pickle.dump(losingRuns, open(losePath, 'w'))

#Load all the variables from
def loadVars():
    varFileNames = ['cards.json', 'relics.json']
    varFilePaths = [None, None]
    varList = [None, None]

    for file in range(len(varFileNames)):
        varFilePaths[file] = os.path.join(pathToData, varFileNames[file])
        varList[file] = json.load(open(varFilePaths[file]))

    #Set Variables to global scope and write cards and relics
    global cards, relics, winningRuns, losingRuns
    cards = varList[0]
    relics = varList[1]

    #Try to load up the wins and loses, if unable to then parse them from dump file
    winsPath = os.path.join(pathToData, 'wins.pkl')
    losePath = os.path.join(pathToData, 'loses.pkl')

    try:
        pickle.load(open("break")) #This is for debugging
        winningRuns = pickle.load(open(winsPath, 'r+'))
        losingRuns = pickle.load(open(losePath, 'r+'))
        print("Run data files found and loaded")
    except IOError:
        print("No run files found, loading from bulk data")
        loadFromFile(winsPath, losePath)

def loadArrays():
    global X, Y
    xPath = os.path.join(pathToData, 'X.npy')
    yPath = os.path.join(pathToData, 'Y.npy')
    try:
        #np.load("break") #For debugging
        # np.load("Break")
        X = np.load(xPath)
        Y = np.load(yPath)
        print("Loaded Arrays from file")
    except IOError:
        print("No array files found, parsing from data files")
        loadVars()
        index = list(cards.keys())
        numCards = len(index)
        runs = winningRuns + losingRuns
        X = np.zeros((len(runs), numCards*2))
        Y = np.concatenate((np.ones(len(winningRuns)), np.zeros(len(losingRuns))))
        np.save(yPath, Y)
        rowCount = 0
        for run in runs:
            #print ('row: ', rowCount)
            for card in run[u'event'][u'master_deck']:
                cardName = str(card)
                #print ('index: ', str(index[collumnCount]))
                #print ('card: ', str(card))
                for i in range(len(index)):
                    indexCardName = str(index[i])
                    upgradedIndexCardName = str(indexCardName + '+1')
                    if cardName == indexCardName:
                        #print cardName, i
                        X[rowCount][i] += 1
                    elif cardName == upgradedIndexCardName:
                         X[rowCount][i+numCards] += 1
            rowCount += 1
        np.save(xPath, X)

def main():
    loadArrays()

if __name__=='__main__':
    loadArrays()


import pygame as pg
import ChessEngine
import os
from itertools import cycle
from copy import deepcopy
pg.init()
pg.mixer.init()

ROOT_DIR = os.path.dirname(__file__)
IMAGE_DIR = os.path.join(ROOT_DIR, 'images')
WIDTH = HEIGHT = 512
DIMENSION = 8
SQ_SIZE = HEIGHT // 8
MAX_FPS = 15
IMAGES = {}
WHITE = (215,215,215)
BLACK = (155,155,155)

def load_images():
    for file in os.listdir(IMAGE_DIR):
        IMAGES[file[:2]] = pg.transform.scale(pg.image.load(os.path.join(IMAGE_DIR, file)), (SQ_SIZE, SQ_SIZE) )

def main():
    screen = pg.display.set_mode((WIDTH,HEIGHT))
    clock = pg.time.Clock()
    game = ChessEngine.GameState()
    load_images()
    running = True
    slctSq = ()
    playerClicks = []
    moves =  []
    while running:
        clock.tick(MAX_FPS)
        for e in pg.event.get():
            if e.type == pg.QUIT:
                running = False

            elif e.type == pg.KEYDOWN:
                if e.key == pg.K_LEFT:
                    game.undo_move()

                if e.key == pg.K_RIGHT:
                    game.redo_move()

                if e.key == pg.K_BACKSPACE:
                    game.unmake_move()

            elif e.type == pg.MOUSEBUTTONDOWN:
                if game.viewmode:
                    game.viewboard = deepcopy(game.board)
                    game.viewmove = len(game.moveLog)
                    game.viewmode = False
                else:
                    location = pg.mouse.get_pos()
                    col = location[0]//SQ_SIZE
                    row = location[1]//SQ_SIZE
                    if slctSq == (row, col):
                        slctSq = ()
                        playerClicks = []
                        moves = []
                    elif slctSq and (row, col) not in moves:
                        slctSq = ()
                        playerClicks = []
                        moves = []
                    elif len(slctSq) == 0 and game.board[row][col] == "--":
                        pass
                    else:
                        if slctSq:
                            slctSq  = (row, col)
                            playerClicks.append(slctSq)
                            pass
                        else:
                            if (game.board[row][col][0] == 'w' and game.whiteToMove) or (game.board[row][col][0] == 'b' and not game.whiteToMove):
                                moves = game.legal_moves(col, row)
                                slctSq  = (row, col)
                                playerClicks.append(slctSq)
                            else:
                                pass

                    if len(playerClicks) == 2:
                        if game.board[playerClicks[0][0]][playerClicks[0][1]] != '--':
                            move = ChessEngine.Move(playerClicks[0], playerClicks[1], game.board)
                            print(move.get_chess_notation())
                            game.make_move(move)
                            slctSq = ()
                            playerClicks = []
                            moves = []
                        else:
                            slctSq = ()
                            playerClicks.pop(0)

        screen.fill(WHITE)
        draw_game(screen, game, moves)
        if playerClicks:
            if game.board[playerClicks[0][0]][playerClicks[0][1]] != '--':
                highlight(screen, playerClicks[0][0],playerClicks[0][1])
        pg.display.flip()

def highlight(screen, row, col):
    s = pg.Surface((SQ_SIZE,SQ_SIZE))  # the size of your rect
    s.set_alpha(128)                # alpha level
    s.fill((0,128,128))           # this fills the entire surface
    screen.blit(s, (col*SQ_SIZE,row*SQ_SIZE))

def draw_game(screen, game, moves):
    draw_board(screen)
    draw_pieces(screen, game)
    draw_moves(screen, moves)

def draw_board(screen):
    tile_color = cycle([WHITE, BLACK])
    for row in range(DIMENSION):
        for col in range(DIMENSION):
            pg.draw.rect(screen, next(tile_color), (col*SQ_SIZE, row*SQ_SIZE, SQ_SIZE,SQ_SIZE))
        next(tile_color)

def draw_pieces(screen, game):
    for row in range(DIMENSION):
        for col in range(DIMENSION):
            if game.viewboard[row][col] != '--':
                screen.blit(IMAGES[game.viewboard[row][col]], (col*64,row*64))

def draw_moves(screen, moves):
    for i in moves:
        pg.draw.circle(screen, (168, 123, 116), (32+i[1]*64, 32+i[0]*64), 5)

if __name__ == '__main__':
    main()


from conv_net.networks.net3 import Network as Net3
from conv_net.networks.net7 import Network as Net7
from conv_net import deep
from read_files import Dataset
from conv_net.utils import log
from conv_net.crossval import crossvalidation
import csv

if __name__ == "__main__":
    log("Reading dataset. ")
    size = 30000
    log("size: %s" % size )
    dataset = Dataset(False, training_set_size=size)

    network = Net7()
    X,y = dataset.read_training_set()
    log("Done")

    log("Training network")
    network.train(X,y)

    log("Predicting test set")

    filename = "submission.csv"
    with open(filename, 'w') as csvfile:
        fieldnames = ["image", "level"]
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        test_set, test_names = dataset.read_test_set("test_set_resize", 100)
        results = network.predict(test_set)
        results_scalar = [result.argmax() for result in results]

        for name, prediction in zip(test_names, results_scalar):
            writer.writerow({'image': name, 'level': prediction})


from rest_framework.decorators import api_view, permission_classes
from rest_framework import generics
from lembaga.models import Lembaga, Institusi, Tema
from lembaga.serializers import LembagaSerializer, InstitusiSerializer, TemaSerializer
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework import viewsets
from django_filters.rest_framework import DjangoFilterBackend

class LembagaView(viewsets.ModelViewSet):
    permission_classes = (AllowAny,)
    serializer_class = LembagaSerializer

    def get_queryset(self, *args, **kwargs):
        """
        Optionally restricts the returned purchases to a given user,
        by filtering against a query parameter in the URL.
        """
        try:
            queryset = Lembaga.objects.all()
            param_institusi = self.request.GET.get('institusi')
            param_tema = self.request.GET.get('tema')
            param_praktikum = self.request.GET.get('praktikum')
            param_nama = self.request.GET.get('nama')

            if param_praktikum:
                temp = param_praktikum.split(',')
                param_praktikum = []
                for i in temp:
                    param_praktikum.append(int(i[-1]))
                queryset = queryset.filter(praktikum_ke__in=param_praktikum)
            if (param_institusi and param_tema):
                param_institusi = param_institusi.split(',')
                param_tema = param_tema.split(',')
                queryset = queryset.filter(institusi__nama__in=param_institusi).filter(tema__nama__in=param_tema)
            else:
                if (param_institusi is not None):
                    param_institusi = param_institusi.split(',')
                    queryset = queryset.filter(institusi__nama__in=param_institusi)
                if (param_tema is not None):
                    param_tema = param_tema.split(',')
                    queryset = queryset.filter(tema__nama__in=param_tema)

            # Find lembaga by Name
            # by performing substring checking
            if param_nama:
                tmp_queryset = queryset
                queryset = []
                for i in tmp_queryset:
                    if param_nama.lower() in i.nama.lower():
                        queryset.append(i)

            return queryset

        # Kasus Error Jika Parameter, Variable, dan Data tidak di set saat melakukan query
        except (ValueError, RuntimeError, TypeError, NameError):
            return {"status": "Harap isi kriteria pencarian"}

class InstitusiView(viewsets.ModelViewSet):
    """
    Provides a get method handler.
    """
    permission_classes = (AllowAny, )
    queryset = Institusi.objects.all()
    serializer_class = InstitusiSerializer

class TemaView(viewsets.ModelViewSet):
    """
    Provides a get method handler.
    """
    permission_classes = (AllowAny, )
    queryset = Tema.objects.all()
    serializer_class = TemaSerializer


def percentage(num_line, den_line):
    num_parts = num_line.split(',')
    den_parts = den_line.split(',')

    #print den_parts[0] + ' , ' + num_parts[0]

    return 100 * float(num_parts[0]) / float(den_parts[0])

# Open the data files
f = open('perf.data', 'r')
cf = open('cache-miss-percentage.data', 'w')
bf = open('branch-miss-percentage.data', 'w')

# Iterator allows for the next() method
f_iter = iter(f)

for line in f_iter:
    # Record the x-axis values
    if ',' not in line and (line.startswith('1') or line.startswith('3')):
        cf.write('\n')
        bf.write('\n')

        cf.write(line.rstrip())
        bf.write(line.rstrip())

        cf.write(' ')
        bf.write(' ')

        cref = next(f_iter)
        cmiss = next(f_iter)
        cpercent = percentage(cmiss, cref)

        bref = next(f_iter)
        bmiss = next(f_iter)
        bpercent = percentage(bmiss, bref)

        cf.write(str(cpercent))
        bf.write(str(bpercent))

        cf.write(' ')
        bf.write(' ')

        continue

    # Otherwise, skip lines that aren't 'cache-references' lines
    if 'cache-references' not in line:
        continue;

    # Grab the cache-misses line and calculcate percentage
    cmiss = next(f_iter)
    #print line + 'hahaaaa'
    #print cmiss
    cpercent = percentage(cmiss, line)

    # Then grab the branch lines and percentage
    bref = next(f_iter)
    bmiss = next(f_iter)
    #print bref + 'loooool'
    #print bmiss
    bpercent = percentage(bmiss, bref)

    cf.write(str(cpercent))
    bf.write(str(bpercent))

    cf.write(' ')
    bf.write(' ')


import os
import tempfile

class WrapStrToFile:
    def __init__(self):
        # здесь инициализируется атрибут filepath, он содержит путь до файла-хранилища
        self._filepath = tempfile.mktemp()
        print(self._filepath)

    @property
    def content(self):
        try:
            with open(self._filepath, "r", encoding="utf-8") as file1:
                return file1.read()
        except FileNotFoundError:
            return "File doesn't exist"

    @content.setter
    def content(self, value):
        with open(self._filepath, "w", encoding="utf-8") as file1:
            file1.write(value)

    @content.deleter
    def content(self):
        os.remove(self._filepath)
        # удаляем файл: os.remove(имя_файла)

wstf = WrapStrToFile()
print(wstf.content)  # Output: File doesn't exist
wstf.content = 'test str'
print(wstf.content)  # Output: test_str
wstf.content = 'text 2'
print(wstf.content)  # Output: text 2
del wstf.content     # после этого файла не существует


import os
from PIL import Image
import PIL.ImageOps
import argparse
import torchvision as V
import matplotlib.pyplot as plt
import ujson as json
import glob
import re
from . import configs
import rich
from rich.progress import track
console = rich.get_console()

def cifar10_burst(dest: str, split: str):
    assert(split in ('train', 'test'))
    if not os.path.exists(dest):
        os.mkdir(dest)
    data = V.datasets.CIFAR10(root=configs.datasets.cifar10.root,
            train=True if split == 'train' else False,
            download=True)
    for i, (x, y) in enumerate(data):
        fpath = os.path.join(dest, '%05d-%1d.png'%(i, y))
        #xinv = PIL.ImageOps.invert(x) # no need to invert like mnist
        #xinv.save(fpath)
        x.save(fpath)
        console.print(fpath)
        print(x, y)
    print(len(data))

def cifar10_collect(src: str):
    '''
    collect processed data from
    '''
    dataset = []
    files = glob.glob(os.path.join(src, '*.json'))
    for file in track(files):
        with open(file, 'rt') as f:
            j = json.load(f)
        label = re.match(r'.*/\d+-(\d).json', file).groups()[0]
        j[0]['label'] = int(label)
        dataset.append(j)
    with open(src+'.json', 'wt') as f:
        json.dump(dataset, f)
    console.print('>_< done')

if __name__ == '__main__':
    ag = argparse.ArgumentParser('python3 -m veccls.cifar10')
    ag.add_argument('action', choices=('burst', 'collect'))
    ag.add_argument('-d', '--destination', default='.', help='dest directory')
    ag.add_argument('-s', '--split', default='train', choices=('train', 'test'))
    ag = ag.parse_args()
    console.print(ag)

    if ag.action == 'burst':
        cifar10_burst(ag.destination, ag.split)
    elif ag.action == 'collect':
        cifar10_collect(ag.destination)


from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor,as_completed,wait
from concurrent import futures
import time
import requests

# 官方文档：https://docs.python.org/zh-cn/3.8/library/concurrent.futures.html#module-concurrent.futures

urls = ['http://www.tencent.com','http://www.google.com/','http://www.kugou.com/']

def jisuan_b(num):
    print("b_time is ：{}。that you num is {}\n".format(time.time(),num))
    return "this is return "

def get_for_content(res):
    r = requests.get(url=res)
    # print(r.text)
    return res

with futures.ThreadPoolExecutor(max_workers=1) as f :
    start_time = time.time()
    # res = f.submit(jisuan_b,'1')          # submit()方法的返回值为一个future对象。此对象可以调用result()方法获取返回值，
    # print(res.result())                 # result函数，实际返回的是回调函数的返回值。result方法本身是阻塞的

    thread_list = []
    for x in urls:
        thread_list.append(f.submit(get_for_content,x))
    # print(wait(thread_list,return_when=FIRST_COMPLETED))              # wait 函数的作用是使主线程阻塞。使满足设定的时候再进行正常执行。
                                            # wait 有三个状态：FIRST_COMPLETED，FIRST_EXCEPTION，ALL_COMPLETED
                                                # FIRST_COMPLETED：只要有一个完成的任务(线程)，就直接的返回，不再进行阻塞。
                                                # FIRST_EXCEPTION：当发生第一个异常的时候退出。如果没有异常的话，就相当于ALL_COMPLETED
                                                # ALL_COMPLETED：当所有的任务(线程)完成的时候退出

    print("执行结束")

    # for x in as_completed(thread_list):     # as_completed函数的作用是接收：executor返回的future对象列表。
                                                # 实际为执行完成的线程返回的furure对象，也就是说安装线程完成的顺序一个一个进行迭代

    #     print("请求的链接为",x.result())     # 然后使主线程阻塞，如果thread_list中有执行完成的线程的话。会yeid这个线程的result对象。直到所有的线程都执行结束为止
    #                                         # 先完成的线程会先通知主线程。(ps.第二个参数为timeout)
    # print("耗时为：{}".format(time.time()-start_time))


# These URLs basically points to MIT's conceptnet5 setup for Web API
# BASE_LOOKUP_URL = 'http://api.conceptnet.io'
# BASE_SEARCH_URL = 'http://api.conceptnet.io/query'
# BASE_ASSOCIATION_URL = 'http://api.conceptnet.io/related'

# if build locall ,use those
BASE_LOOKUP_URL = 'http://127.0.0.1:8084'
BASE_SEARCH_URL = 'http://127.0.0.1:8084/query'
BASE_ASSOCIATION_URL = 'http://127.0.0.1:8084/related'

# This is the supported query arguments for LookUp API
# :param offset: skip the specified amount of first results
# :type offset: integer
# :param limit: change the number of results from the default of 50
# :type limit: integer

SUPPORTED_LOOKUP_ARGS = ['offset', 'grouped', 'limit']

# This is the supported query arguments for Association API
# :param limit: change the number of results from the default of 50
# :type limit: integer
# :param filter: return only results that start with the given URI. For example,
#                filter=/c/en returns results in English.
# :type filter: a uri, e.g. '/c/en/cat' (Different than lookup API!)
SUPPORTED_ASSOCIATION_ARGS = ['filter', 'limit']

# Supported arguments for Search API
# :param {rel, start, end, node, source}: giving a ConceptNet URI for any of these
#       parameters will return edges whose corresponding fields start with the given path
# :type {rel, start, end, node, source}: uri
# :param node: returns edges whose rel, start, or end start with the given URI
# :type node: uri
# :param limit: change the number of results from the default of 50
# :type limit: integer
# :param filter: If 'core', only get edges from the ConceptNet 5 Core (not from ShareAlike resources),
#       if 'core-assetions', search for edges by default, and there can be many edges representing the same assertion.
# :type filter: Either 'core' or 'core-assertions'
SUPPORTED_SEARCH_ARGS = ['start', 'end', 'rel', 'node', 'sources', 'other', 'limit', 'filter']

# I filted out my instered Chinese commonsense.

intered_lang = 'zh'

# Baidu translate service settings
base_url = 'http://api.fanyi.baidu.com/api/trans/vip/translate'
appid = 'your appid'
secretKey = 'your secretKey'

LCODE_ALIASES = {
    # Pretend that various Chinese languages and variants are equivalent. This
    # is linguistically problematic, but it's also very helpful for aligning
    # them on terms where they actually are the same.
    #
    # This would mostly be a problem if ConceptNet was being used to *generate*
    # Chinese natural language text.
    'cmn': 'zh',
    'yue': 'zh',
    'zh_tw': 'zh',
    'zh_cn': 'zh',
    'zh-tw': 'zh',
    'zh-cn': 'zh',

    'nds-de': 'nds',
    'nds-nl': 'nds',

    # An easier case: consider Bahasa Indonesia and Bahasa Malay to be the
    # same language, with code 'ms', because they're already 90% the same.
    # Many sources use 'ms' to represent the entire macrolanguage, with
    # 'zsm' to refer to Bahasa Malay in particular.
    'zsm': 'ms',
    'id': 'ms',

    # We had to make a decision here on Norwegian. Norwegian Bokmål ('nb') and
    # Nynorsk ('nn') have somewhat different vocabularies but are mutually
    # intelligible. Informal variants of Norwegian, especially when spoken,
    # don't really distinguish them. Some Wiktionary entries don't distinguish
    # them either. And the CLDR data puts them both in the same macrolanguage
    # of Norwegian ('no').
    #
    # The catch is, Bokmål and Danish are *more* mutually intelligible than
    # Bokmål and Nynorsk, so maybe they should be the same language too. But
    # Nynorsk and Danish are less mutually intelligible.
    #
    # There is no language code that includes both Danish and Nynorsk, so
    # it would probably be inappropriate to group them all together. We will
    # take the easy route of making the language boundaries correspond to the
    # national boundaries, and say that 'nn' and 'nb' are both kinds of 'no'.
    #
    # More information: http://languagelog.ldc.upenn.edu/nll/?p=9516
    'nn': 'no',
    'nb': 'no',

    # Our sources have entries in Croatian, entries in Serbian, and entries
    # in Serbo-Croatian. Some of the Serbian and Serbo-Croatian entries
    # are written in Cyrillic letters, while all Croatian entries are written
    # in Latin letters. Bosnian and Montenegrin are in there somewhere,
    # too.
    #
    # Applying the same principle as Chinese, we will unify the language codes
    # into the macrolanguage 'sh' without unifying the scripts.
    'bs': 'sh',
    'hr': 'sh',
    'sr': 'sh',
    'hbs': 'sh',
    'sr-latn': 'sh',
    'sr-cyrl': 'sh',

    # More language codes that we would rather group into a broader language:
    'arb': 'ar',  # Modern Standard Arabic -> Arabic
    'arz': 'ar',  # Egyptian Arabic -> Arabic
    'ary': 'ar',  # Moroccan Arabic -> Arabic
    'ckb': 'ku',  # Central Kurdish -> Kurdish
    'mvf': 'mn',  # Peripheral Mongolian -> Mongolian
    'tl': 'fil',  # Tagalog -> Filipino
    'vro': 'et',  # Võro -> Estonian
    'sgs': 'lt',  # Samogitian -> Lithuanian
    'ciw': 'oj',  # Chippewa -> Ojibwe
    'xal': 'xwo',  # Kalmyk -> Oirat
    'ffm': 'ff',  # Maasina Fulfulde -> Fula
}


# -*- coding: utf-8 -*-
"""
Created on Tue Aug 24 19:55:28 2021

@author: jon-f
"""

import numpy as np
import matplotlib.pyplot as plt
import os
from skimage.transform import resize

def binary_array_to_hex(arr):

	bit_string = ''.join(str(int(b)) for b in arr.flatten())
	width = int(np.ceil(len(bit_string)/4))
	return '{:0>{width}x}'.format(int(bit_string, 2), width=width)

def image_to_edgehex(numpy_image):
    numpy_array = resize(numpy_image, (64, 64))

    size = numpy_array.shape
    grayscale = np.zeros(size[:-1])
    grayscale1der = np.zeros(size[:-1])

    for i in range(size[0]):
        for j in range(size[1]):

            if size[2] == 4:
                val = numpy_array[i, j]
                n_val = (0.299*val[0]+0.587*val[1]+0.114*val[2])*val[3]
            elif size[2] == 3:
                val = numpy_array[i, j]
                n_val = 0.299*val[0]+0.587*val[1]+0.114*val[2]
            else:
                n_val = numpy_array[i, j]

            grayscale[i, j] = n_val

            if i == 0 or j == 0:
                continue

            sub = grayscale[i-1:i+1, j-1:j+1]

            dIdx = 1/2 * ((sub[0, 1] - sub[0, 0]) + (sub[1, 1] - sub[1, 0]))
            dIdy = 1/2 * ((sub[1, 0] - sub[0, 0]) + (sub[1, 1] - sub[0, 1]))

            dIdp = abs(dIdx/2 + dIdy/2)
            if dIdp < 0.03:
                continue

            grayscale1der[i, j] = 1

    return binary_array_to_hex(grayscale1der)

def dilate(image_file):
    numpy_array = resize(image_file, (64, 64))

    size = numpy_array.shape[:-1]
    dilated = np.zeros(size)

    for i in range(size[0]):
        for j in range(size[1]):

            sub = image_file[i-1:i+1, j-1:j+1]

            if (sub == 1).any():
                dilated[i, j] = 1
    return dilated

def hamming_distance(hash1, hash2):
    dist = sum(n1 != n2 for n1, n2 in zip(hash1, hash2))
    return dist

def hash_match(hash1, hash2, percent_match=60):
    # returns True if at least percent_match %
    return hamming_distance(hash1, hash2)/len(hash1) < percent_match/100


#!/usr/bin/env pythonimport coliche, os
import bn.bn

def bn2dot(bnfile, outfile, vdfile=None, loners=False, center=None,
           awfile=None):
    # give None to outfile to get string back

    dotbuffer = []

    bns = bn.bn.load(bnfile, False)
    varc = bns.varc
    arcs = bns.arcs()

    names = vdfile \
            and list(l.split("\t",1)[0] for l in file(vdfile)) \
            or map(str, range(varc))

    lonerset = range(varc)

    if center:
        try:
            center = int(center)
        except:
            center = names.index(center)

    showvars = bns.mbnodes(center) if center != None else set(range(varc))

    aws = {}
    if awfile != None:
        for l in file(awfile):
            t = l.split()
            x,y = map(int,t[0:2])
            w = float(t[2])
            aws[(x,y)]=w

    dotbuffer.append("digraph BN {")

    for x,y in arcs:
        if x in showvars and y in showvars:
            wstr = ''
            if (x,y) in aws:
                wstr = ' [label="%.2g"]' % aws[(x,y)]

            nx, ny = names[x], names[y]
            dotbuffer.append('  "%s" -> "%s"%s;' % (nx, ny,wstr))

        if x in lonerset:
            lonerset.remove(x)
        if y in lonerset:
            lonerset.remove(y)

    if loners:
        for l in sorted(lonerset):
            if l in showvars: dotbuffer.append('"%s";' % names[l])

    dotbuffer.append("}")
    dotstr = '\n'.join(dotbuffer)
    if outfile:
        file(outfile,"w").write(dotstr)
    else:
        return dotstr

if __name__ == "__main__":
    coliche.che(bn2dot,
                """bnfile; outfile
                -n --vdfile vdfile     : to add names to the picture
                -l --loners            : show orphan nodes too
                --mb center            : draw only markov blanket of center
                -w --arcweights awfile : draw arcweights
                """)


from django.shortcuts import render, get_object_or_404
from .models import Post, Category, User
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from taggit.models import Tag
from django.db.models import Count
from django.utils import timezone

def post_list(request, tag_slug=None):
    posts = Post.objects.filter(status="published")
    latest_posts = Post.objects.filter(published_date__lte=timezone.now()).reverse()[:3]

    # post tag
    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        posts = posts.filter(tags__in=[tag])

    paginator = Paginator(posts, 5)  # change this to accommodate number of posts per page
    page = request.GET.get('page', 1)
    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'posts': posts,
        'page': page,
        'tag': tag,
        'latest_posts': latest_posts,
    }

    return render(request, 'blog/post_list.html', context, )

def author_post_list(request, author):
    author = User.objects.get(username=author)

    posts_list = author.posts.filter(status="published")

    paginator = Paginator(posts_list, 3)  # change this to accommodate number of posts per page
    page = request.GET.get('page', 1)

    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'author': author,
        'posts': posts,
        'page': page
    }

    return render(request, 'blog/author_post_list.html', context)

def category_post_list(request, slug):
    category = Category.objects.get(slug=slug)

    posts_list = category.posts.filter(status="published")
    paginator = Paginator(posts_list, 3)  # change this to accommodate number of posts per page

    page = request.GET.get('page', 1)

    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'category': category,
        'posts': posts,
        'page': page
    }

    return render(request, 'blog/category.html', context)

def blog_post(request, slug):
    # post = Post.objects.get(slug=slug)
    post = get_object_or_404(Post, slug=slug, status='published')

    # List of similar posts
    post_tags_ids = post.tags.values_list('id', flat=True)
    similar_posts = Post.objects.filter(active=True, tags__in=post_tags_ids).exclude(id=post.id)
    similar_posts = similar_posts.annotate(same_tags=Count('tags')).order_by('-same_tags', '-active')[
                    :3]  # adjust slice here for amount of posts to show

    context = {
        'post': post,
        'similar_posts': similar_posts,
    }

    return render(request, 'blog/blog_post.html', context)


"""Platform for sensor integration."""

import logging

from datetime import datetime

from .const import (
    DOMAIN,
    MODEM_GATEWAY,
    EVT_MODEM_CONNECTED,
    EVT_MODEM_DISCONNECTED,
    EVT_LTE_CONNECTED,
    EVT_LTE_DISCONNECTED,
    SENSOR_LASTUPD
)

from homeassistant.components.binary_sensor import (
    DEVICE_CLASS_CONNECTIVITY,
    BinarySensorEntity,
)

_LOGGER = logging.getLogger(__name__)

SENSOR_SIGNAL = 'signal_strength'
SENSOR_STATUS = 'modem_status'
SENSOR_OPERATOR = 'cell_operator'
REGISTERED_STATUS = 'registered'
CONNECTED_STATUS = 'connected'
GSM_SENSOR_ID = 'mm_modem.signal_strength'
GSM_SENSOR_NAME = 'GSM Modem'
LTE_SENSOR_ID = 'mm_modem.lte_status'
LTE_SENSOR_NAME = 'LTE Connection'

def setup_platform(hass, config, add_entities, discovery_info=None):
    """Set up the sensor platform."""
    add_entities([GsmModemSensor(hass)])
    add_entities([LteConnectionSensor(hass)])

async def async_setup_entry(hass, config_entry, async_add_entities):
    """Set up the sensors."""
    async_add_entities([GsmModemSensor(hass)])
    async_add_entities([LteConnectionSensor(hass)])

class GsmModemSensor(BinarySensorEntity):
    """Representation of a Sensor."""
    _signal_strength = 0
    _modem_status = 'none'
    _cell_operator = 'none'
    _hass = None
    _prev_status = False

    def __init__(self, hass):
        """Initialize the sensor."""
        self._state = None
        self._hass = hass
        hass.bus.async_listen(EVT_MODEM_DISCONNECTED,
                              self._handle_modem_disconnected)

        hass.bus.async_listen(EVT_MODEM_CONNECTED,
                              self._handle_modem_connected)
        self.update()

    async def _handle_modem_disconnected(self, call):
        self.update()
        self.async_write_ha_state()

    async def _handle_modem_connected(self, call):
        self.update()
        self.async_write_ha_state()

    def get_gateway(self):
        """Returns the modem gateway instance from hass scope"""
        return self._hass.data[DOMAIN][MODEM_GATEWAY]

    @property
    def name(self):
        """Return the name of the sensor."""
        return GSM_SENSOR_NAME

    @property
    def is_on(self):
        """Return the state of the sensor."""
        return [REGISTERED_STATUS,
                CONNECTED_STATUS].__contains__(self._modem_status)

    @property
    def device_class(self):
        """Return the class of this device, from component DEVICE_CLASSES."""
        return DEVICE_CLASS_CONNECTIVITY

    @property
    def unique_id(self):
        """Return a unique ID."""
        return GSM_SENSOR_ID

    @property
    def extra_state_attributes(self):
        """Return device specific state attributes.
        Implemented by platform classes. Convention for attribute names
        is lowercase snake_case.
        """
        return {SENSOR_STATUS: self._modem_status,
                SENSOR_OPERATOR: self._cell_operator,
                SENSOR_SIGNAL: self._signal_strength,
                SENSOR_LASTUPD: datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}

    def update(self):
        """Fetch new state data for the sensor.

        This is the only method that should fetch new data for Home Assistant.
        """
        gateway = self.get_gateway()
        modem_info = gateway.get_modem_state()
        if modem_info is not None:
            self._signal_strength = modem_info['signal']
            self._cell_operator = modem_info['operator']
            self._modem_status = modem_info['status']
            if not self._prev_status:
                self._prev_status = True
                _LOGGER.info('GSM modem connected')
        else:
            self._signal_strength = 0
            self._modem_status = 'none'
            self._cell_operator = 'none'
            if self._prev_status:
                self._prev_status = False
                _LOGGER.info('GSM modem disconnected')

class LteConnectionSensor(BinarySensorEntity):
    """Representation of a Sensor."""
    _modem_status = None
    _hass = None
    _prev_status = False

    def __init__(self, hass):
        """Initialize the sensor."""
        self._state = None
        self._hass = hass
        hass.bus.async_listen(EVT_LTE_DISCONNECTED,
                              self._handle_lte_disconnected)

        hass.bus.async_listen(EVT_LTE_CONNECTED,
                              self._handle_lte_connected)

    async def _handle_lte_disconnected(self, call):
        self.update()
        self.async_write_ha_state()

    async def _handle_lte_connected(self, call):
        self.update()
        self.async_write_ha_state()

    def get_gateway(self):
        """Returns the modem gateway instance from hass scope"""
        return self._hass.data[DOMAIN][MODEM_GATEWAY]

    @property
    def name(self):
        """Return the name of the sensor."""
        return LTE_SENSOR_NAME

    @property
    def is_on(self):
        """Return the state of the sensor."""
        return [CONNECTED_STATUS].__contains__(self._modem_status)

    @property
    def device_class(self):
        """Return the class of this device, from component DEVICE_CLASSES."""
        return DEVICE_CLASS_CONNECTIVITY

    @property
    def unique_id(self):
        """Return a unique ID."""
        return LTE_SENSOR_ID

    @property
    def extra_state_attributes(self):
        """Return device specific state attributes.
        Implemented by platform classes. Convention for attribute names
        is lowercase snake_case.
        """
        return {}

    def update(self):
        """Fetch new state data for the sensor.
        This is the only method that should fetch new data for Home Assistant.
        """
        gateway = self.get_gateway()
        modem_info = gateway.get_modem_state()
        if modem_info is not None:
            self._modem_status = modem_info['status']
        else:
            self._modem_status = 'none'


#every day coding test 2018.07.09 [!]
'''
Q : Given an integer N, find the number of possible balanced parentheses with N opening and closing brackets.
'''

import copy

# DUMMY = [4]
DUMMY = [1, 2, 3]
G_list = []
G_Question = None

def combination_rule(past_list):
    '''
     1. just condition check.
       1. exit condition.
       2. can be add 1
       3. can be add 2
       4. can be add both.
        1. 1 > 2
            both
        2. 1 == 2
            only 1
    '''
    if past_list.count(1) == G_Question:
        if past_list.count(2) == G_Question:
            G_list.append(map(lambda x: '(' if x==1 else ')', past_list))
            return
        else:
            past_list.append(2)
            combination_rule(past_list)
    else:
        cron_list = copy.copy(past_list)
        if past_list.count(1) > past_list.count(2):
            cron_list.append(2)
            combination_rule(cron_list)
        past_list.append(1)
        combination_rule(past_list)

if __name__ == '__main__':
    for sample in DUMMY:
        G_list = []
        G_Question = sample
        combination_rule([1])
        print ('%s --> %s --> %s') % (G_Question, G_list, len(G_list))


import cv2
import numpy as np
import matplotlib.pyplot as plt

img1 = cv2.imread('shanghai-11.png')
img2 = cv2.imread('shanghai-12.png')
img3 = cv2.imread('shanghai-13.png')

plt.figure(1)
plt.subplot(131), plt.imshow(img1, cmap='gray'), plt.title("Image 1")
plt.subplot(132), plt.imshow(img2, cmap='gray'), plt.title("Image 2")
plt.subplot(133), plt.imshow(img3, cmap='gray'), plt.title("Image 3")
plt.show()

orbObj = cv2.ORB_create()  # Create ORB Object
kp1, descriptors1 = orbObj.detectAndCompute(img1, None)  # Computing descriptors and keypoints
kp2, descriptors2 = orbObj.detectAndCompute(img2, None)
kp3, descriptors3 = orbObj.detectAndCompute(img3, None)

matcher = cv2.DescriptorMatcher_create(cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING)  # Creating Matcher Object
GOOD_MATCH_PERCENT = 0.15
MAX_MATCHES = 25

# Image 1 - Image 2 Matcher
matches1 = matcher.match(descriptors1, descriptors2, None)
matches1 = sorted(matches1, key=lambda x: x.distance, reverse=False)
good_matches1 = int(len(matches1) * GOOD_MATCH_PERCENT)
matches1 = matches1[:good_matches1]  # Remove weak matches

# Image 1 - Image 3 Matcher
matches2 = matcher.match(descriptors1, descriptors3, None)
matches2 = sorted(matches2, key=lambda x: x.distance, reverse=False)
good_matches2 = int(len(matches2) * GOOD_MATCH_PERCENT)
matches2 = matches2[:good_matches2]  # Remove weak matches

# Image 2 - Image 3 Matcher
matches3 = matcher.match(descriptors2, descriptors3, None)
matches3 = sorted(matches3, key=lambda x: x.distance, reverse=False)
good_matches3 = int(len(matches3) * GOOD_MATCH_PERCENT)
matches3 = matches3[:good_matches3]  # Remove weak matches

# Drawing Matches
img_matches1 = cv2.drawMatches(img1, kp1, img2, kp2, matches1[0:MAX_MATCHES], None)
img_matches2 = cv2.drawMatches(img1, kp1, img3, kp3, matches2[0:MAX_MATCHES], None)
img_matches3 = cv2.drawMatches(img2, kp2, img3, kp3, matches3[0:MAX_MATCHES], None)

plt.figure(2)
plt.subplot(311), plt.imshow(img_matches1, cmap='gray'), plt.title("Image 1 - Image 2")
plt.subplot(312), plt.imshow(img_matches2, cmap='gray'), plt.title("Image 1 - Image 3")
plt.subplot(313), plt.imshow(img_matches3, cmap='gray'), plt.title("Image 2 - Image 3")
plt.show()

# Best Match Result : Img1 - Img2
points1 = np.zeros((len(matches1), 2), dtype=np.float32)
points2 = np.zeros((len(matches1), 2), dtype=np.float32)

for i, match in enumerate(matches1):
    points1[i, :] = kp1[match.queryIdx].pt
    points2[i, :] = kp2[match.trainIdx].pt

h, mask = cv2.findHomography(points2, points1, cv2.RANSAC)  # Compute Homography
img1_height, img1_width, img1_channels = img1.shape
img2_height, img2_width, img2_channels = img2.shape
im2Aligned = cv2.warpPerspective(img2, h, (img2_width + img1_width, img2_height))  # Align Img2

stitched_Image12 = np.copy(im2Aligned)  # Stitch Img 1 with Aligned Img 2
stitched_Image12[0:img1_height, 0:img1_width] = img1

plt.figure(3)
plt.imshow(stitched_Image12, cmap='gray'), plt.title("Image 1 - Image 2 Stiched")
plt.show()

# Repeating the above steps for Stiching: stitched_Image12 - Image 3
stitched_Image12 = stitched_Image12[:, :1343]
kp12, descriptors12 = orbObj.detectAndCompute(stitched_Image12, None)
matches12 = matcher.match(descriptors12, descriptors3, None)
matches12 = sorted(matches12, key=lambda x: x.distance, reverse=False)
good_matches12 = int(len(matches12) * GOOD_MATCH_PERCENT)
matches12 = matches12[:good_matches12]  # Remove weak matches

# Drawing Matches
img_matches12 = cv2.drawMatches(stitched_Image12, kp12, img3, kp3, matches12[0:MAX_MATCHES], None)
plt.figure(4)
plt.imshow(img_matches12, cmap='gray'), plt.title("Stiched Image 12 - Image 3 Matches")
plt.show()

# Stiching Images:

points1 = np.zeros((len(matches12), 2), dtype=np.float32)
points2 = np.zeros((len(matches12), 2), dtype=np.float32)

for i, match in enumerate(matches12):
    points1[i, :] = kp12[match.queryIdx].pt
    points2[i, :] = kp3[match.trainIdx].pt

h123, mask123 = cv2.findHomography(points2, points1, cv2.RANSAC)  # Compute Homography
img12_height, img12_width, img12_channels = stitched_Image12.shape
img3_height, img3_width, img3_channels = img3.shape
im3Aligned = cv2.warpPerspective(img3, h123, (img12_width + img3_width, img3_height))
stitched_Image123 = np.copy(im3Aligned)  # Stitch Img 12 with Aligned Img 3
stitched_Image123[0:img12_height, 0:img12_width] = stitched_Image12

plt.figure(5)
plt.imshow(stitched_Image123, cmap='gray'), plt.title("Image 1 - Image 2 - Image 3 Stiched")
plt.show()

stitched_Image123_noblck = np.copy(stitched_Image123)
stitched_Image123_noblck = stitched_Image123_noblck[:, :1830]
cv2.imwrite('stiched_image.jpg', stitched_Image123_noblck, [cv2.IMWRITE_JPEG_QUALITY, 80])

plt.figure(6)
plt.subplot(2, 1, 1), plt.imshow(stitched_Image123, cmap='gray'), plt.title("Image 1 - Image 2 - Image 3 Stiched")
plt.subplot(2, 1, 2), plt.imshow(stitched_Image123_noblck, cmap='gray'), plt.title("No black -Image 1 - Image 2 - "
                                                                                "Image 3 Stiched")
plt.show()

# SECTION 2

obj = cv2.imread('object.jpg')
img1 = cv2.imread('clutter01.jpg')
img2 = cv2.imread('clutter02.jpg')
img3 = cv2.imread('clutter03.jpg')
img4 = cv2.imread('clutter04.jpg')

plt.figure(1)
plt.subplot(151), plt.imshow(img1[:, :, ::-1]), plt.title("Clutter 1")
plt.subplot(152), plt.imshow(img2[:, :, ::-1]), plt.title("Clutter 2")
plt.subplot(153), plt.imshow(img3[:, :, ::-1]), plt.title("Clutter 3")
plt.subplot(154), plt.imshow(img4[:, :, ::-1]), plt.title("Clutter 4")
plt.subplot(155), plt.imshow(obj[:, :, ::-1]), plt.title("Object")
plt.show()

# Computing descriptors and keypoints using SIFT
siftObj = cv2.SIFT_create()  # Create SIFT Object
kp1, descriptors1 = siftObj.detectAndCompute(img1, None)
kp2, descriptors2 = siftObj.detectAndCompute(img2, None)
kp3, descriptors3 = siftObj.detectAndCompute(img3, None)
kp4, descriptors4 = siftObj.detectAndCompute(img4, None)
kp5, descriptors5 = siftObj.detectAndCompute(obj, None)

# Too many descriptors and keypoints => define function

def flann_with_sift(image1, d1, keyp1, image2, d2, keyp2):
    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
    search_params = dict(checks=100)
    flann_obj = cv2.FlannBasedMatcher(index_params, search_params)
    matches = flann_obj.knnMatch(d1, d2, 2)

    # Filter matches with Lowe's ratio test(src: opencv.com - tutorial_feature_flann_matcher)
    thresh = 0.67
    valid_matches = []
    for m, n in matches:
        if m.distance < thresh * n.distance:
            valid_matches.append(m)

    if len(valid_matches) < 10:
        print(f"Valid matches: {len(valid_matches)} - Not enough matches")
    else:
        img_matched = np.empty((max(image1.shape[0], image2.shape[0]), image1.shape[1] + image2.shape[1], 3),
                               dtype=np.uint8)
        cv2.drawMatches(image1, keyp1, image2, keyp2, valid_matches, img_matched,
                        flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
        plt.figure(), plt.imshow(img_matched[:, :, ::-1]), plt.title(
            f"Matched images with Flann and SIFT")  # Draw Matches

        points1 = np.zeros((len(valid_matches), 2), dtype=np.float32)
        points2 = np.zeros((len(valid_matches), 2), dtype=np.float32)
        for i, match in enumerate(valid_matches):
            points1[i, :] = keyp1[match.queryIdx].pt
            points2[i, :] = keyp2[match.trainIdx].pt
        h, mask = cv2.findHomography(points2, points1, cv2.RANSAC)
        imHeight, imWidth, channels = image1.shape
        imAligned = cv2.warpPerspective(image2, h, (imWidth, imHeight))
        #Draw Rectangles
        cont_image = np.copy(imAligned)
        cont_image = cv2.cvtColor(cont_image, cv2.COLOR_BGR2GRAY)
        contours, hierarchy = cv2.findContours(cont_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        cv2.drawContours(image1, contours, -1, (0, 0, 255), 5)
        plt.figure()
        plt.subplot(1, 2, 1), plt.imshow(image1[:, :, ::-1]), plt.title("Original")
        plt.subplot(1, 2, 2), plt.imshow(imAligned[:,:,::-1]), plt.title("Homography")
        plt.show()

flann_with_sift(img1, descriptors1, kp1, obj, descriptors5, kp5)
flann_with_sift(img2, descriptors2, kp2, obj, descriptors5, kp5)
flann_with_sift(img3, descriptors3, kp3, obj, descriptors5, kp5)
flann_with_sift(img4, descriptors4, kp4, obj, descriptors5, kp5)


from typing import List
"""
Given an integer array nums, find the contiguous subarray (containing at least one number) which has the largest sum and return its sum.

A subarray is a contiguous part of an array.
"""

class Solution:
    def maxSubArray(self, nums: List[int]) -> int:
        dp = [nums[0]]
        # result = float("-inf")
        for i in range(1, len(nums)):
            dp.append(nums[i] + (dp[i-1] if dp[i-1] > 0 else 0))

        return max(dp)
nums = [-2, 1, -3, 4, -1, 2, 1, -5, 4] # [5,4,-1,7,8]
sol = Solution()
print(sol.maxSubArray(nums))
# Input: nums = [-2, 1, -3, 4, -1, 2, 1, -5, 4]
# Output: 6
# Explanation: [4, -1, 2, 1] has the largest sum = 6.


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/11/22 12:54
# @Author  : Ryu
# @Site    :
# @File    : yibu.py
# @Software: PyCharm

import asyncio,aiohttp
import time
import requests

async def f1(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            print(await resp.text())

def timecount(fun):
    def a(url):
        start=time.clock()
        fun(url)
        end=time.clock()
        print(end-start)
    return a

@timecount
def f2(url):
    for i in range(5):
        print(requests.get(url).text)

url='http://httpbin.org/ip'

# f2(url)
# time1=time.clock()
# loop=asyncio.get_event_loop()
# task=[f1(url) for i in range(600)]
# loop.run_until_complete(asyncio.wait(task))
# print(time.clock()-time1)


import openpyxl
import os, string, sys
from lib import l

DIRS_SOCIUM = ['/media/da3/asteriskBeagleAl/Socium/2017/', '/media/da3/asteriskBeagleAl/Socium/2018/']

def isSNILS(snils):
    if snils != None:
        t = str(snils).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        if len(t) >= 11:
            if t[3] == '-' and t[7] == '-' and t[11] == ' ':
                return True
            else:
                return False
        else:
            return False
    return False

def isAudio(audio):
    if audio != None:
        t = str(audio).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        t1 = t.split('/')[len(t.split(('/'))) - 1]
        if len(t1) > 26:
            if t1[2] == '.' and t1[5] == '.' and t1[10] == '_' and t1[13] == '-' and t1[16] == '-':
                return ['длинный', t1]
            elif len(''.join([char for i, char in enumerate(t1) if char in string.digits and i < 26])) == 25 \
                    and t1[14] == '_':
                return ['короткий', t1]
            elif t.endswith('.mp3') or t.endswith('.wav'):
                return ['расширение', t1]
            else:
                return ['', audio]
        else:
            return ['', audio]
    return ['', audio]

def isSocium(audio):
    if audio != None:
        t = str(audio).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        t1 = t.split('/')[len(t.split(('/'))) - 1]
        if len(t1) > 26:
            if t1[2] == '.' and t1[5] == '.' and t1[10] == '_' and t1[13] == '-' and t1[16] == '-' \
                    and (t1[6:10] == 2017 or t1[6:10] == 2018):
                return True
            elif len(''.join([char for i, char in enumerate(t1) if char in string.digits and i < 26])) == 25 \
                    and t1[14] == '_' and (t1[:4] == 2017 or t1[:4] == 2018):
                return True
            else:
                return False
        else:
            return False
    return False

snils_audios = {}
snils_audios_fullpath = {}

for dir_socium in DIRS_SOCIUM:
    directories = os.listdir(dir_socium)
    for directory in directories:
        files = os.listdir(dir_socium + directory)
        for file in files:
            if file.endswith('.xlsx'):
                wb = openpyxl.load_workbook(filename=dir_socium + directory + '/'+ file, read_only=True)
                for sheetname in wb.sheetnames:
                    sheet = wb[sheetname]
                    if not sheet.max_row:
                        print('Файл', file, 'Excel некорректно сохранен OpenPyxl. Откройте и пересохраните его')
                        continue
                    print('\t накоплено связей СНИЛС-audio:', len(snils_audios_fullpath),'\n', dir_socium + directory +
                          '/'+ file + '!' + sheetname)
                    # В каждой строчке определяем где есть аудифайл и СНИЛС
                    table_j_end = 0  # Если больше 10 пустых ячеек - на следующую срочку
                    table_k_end = 0  # Если больше 10 пустых строчек - заканчиваем чтение таблицы
                    for j, row in enumerate(sheet.rows):
                        if table_j_end == 10 and j == 10:
                            break
                        snils = 0
                        audofiles = []
                        audofileExt = []
                        for k, cell in enumerate(row):
                            if cell.value != None:
                                table_j_end = 0
                                table_k_end = 0
                            else:
                                table_j_end += 1
                                table_k_end += 1
                            if table_k_end > 10:
                                break
                            if isSNILS(cell.value):
                                snils = l(cell.value)
                            else:
                                rezAudio = isAudio(cell.value)
                                if rezAudio[0]:
                                    if rezAudio[1].endswith('.wav') or rezAudio[1].endswith('.mp3'):
                                        rezAudioName = rezAudio[1][:-4]
                                    elif rezAudio[1].endswith('.') or rezAudio[1].endswith('/'):
                                        rezAudioName = rezAudio[1][:-1]
                                    else:
                                        rezAudioName = rezAudio[1]
                                    for audofileTek in files:
                                        if audofileTek[:-4] == rezAudioName:
                                            audofileExt.append(audofileTek)
                                            audofiles.append(rezAudioName)
                                            break
                        if snils and len(audofiles):
                            for i, audofile in enumerate(audofiles):
                                if snils_audios.get(snils, None):
                                    if audofile not in snils_audios[snils]:
                                        snils_audios[snils].append(audofile)
                                        snils_audios_fullpath[snils].append(dir_socium + directory + '/'+ audofileExt[i])
                                    else:
                                        print('\tДля СНИЛСа', snils, 'уже есть', dir_socium + directory + '/' + audofileExt[i])
                                else:
                                    snils_audios[snils] = [audofile]
                                    snils_audios_fullpath[snils] = [dir_socium + directory + '/'+ audofileExt[i]]
                        else:
                            if not snils and not len(audofiles):
                                pass
                            elif len(audofiles):
                                for audofile in audofiles:
                                    print('\tНе нашлось СНИЛСа для:', dir_socium + directory + '/' + audofileExt[i])
                            elif snils:
                                #print('\tВ директории', dir_socium + directory,'Не нашлось аудиофайла для СНИЛСа:', snils)
                                pass


import numpy as np
from scipy.spatial.distance import cdist
from scipy.optimize import linprog
from functools import partial
import itertools

def sparse_jump(Y, n_states, max_features, jump_penalty=1e-5,
                max_iter=10, tol=1e-4, n_init=10, verbose=False):
    # Implementation of sparse jump model
    n_obs, n_features = Y.shape
    max_features = np.clip(max_features, a_min=1, a_max=np.sqrt(n_features))
    feat_w = np.repeat(1 / np.sqrt(n_features), n_features)
    states = None

    for it in range(max_iter):
        print(feat_w)
        states, _ = jump(Y * np.sqrt(feat_w),
                      n_states,
                      initial_states=states,
                      jump_penalty=jump_penalty,
                      n_init=n_init)
        if len(np.unique(states)) == 1:
            break
        else:
            new_w = get_weights(Y, states, max_features, n_states)
        if abs(new_w - feat_w).sum() / abs(feat_w).sum() < tol:
            break
        elif verbose:
            print('Iteration {}, w diff {:.6e}'.format(it, abs(new_w - feat_w).sum()))
        feat_w = new_w

    return states, feat_w

def sparse_probabilistic_jump(Y, n_states, order, disc_size,
                              max_features, jump_penalty=1e-5,
                max_iter=10, tol=1e-4, n_init=10, verbose=False):
    # Implementation of sparse jump model
    n_obs, n_features = Y.shape
    max_features = np.clip(max_features, a_min=1, a_max=np.sqrt(n_features))
    feat_w = np.repeat(1 / np.sqrt(n_features), n_features)
    states = None

    for it in range(max_iter):
        states = probabilistic_jump(Y * np.sqrt(feat_w),
                      n_states,  order, disc_size,
                      initial_states=states,
                      jump_penalty=jump_penalty,
                      n_init=n_init)
        if n_states - np.count_nonzero(states.sum(axis = 0) == 0) == 1:
            break
        else:
            new_w = get_probabilistic_weights(Y, states, max_features, n_states)
        if abs(new_w - feat_w).sum() / abs(feat_w).sum() < tol:
            break
        elif verbose:
            print('Iteration {}, w diff {:.6e}'.format(it, abs(new_w - feat_w).sum()))
        feat_w = new_w

    return states, feat_w

def jump(Y, n_states, jump_penalty=1e-5, initial_states=None,
         max_iter=10, n_init=10, tol=None, verbose=False):
    # Fit jump model using framework of Bemporad et al. (2018)
    if initial_states is not None:
        initial_states = np.array(initial_states, dtype=np.int64)
        if len(np.unique(initial_states)) == n_states:
            s = initial_states.copy()
        else:
            s = init_states(Y, n_states)
    else:
        s = init_states(Y, n_states)

    n_obs, n_features = Y.shape
    Gamma = jump_penalty * (1 - np.eye(n_states))
    best_loss = None
    best_s = None

    for init in range(n_init):
        mu = np.zeros((n_states, n_features))
        loss_old = 1e10
        for it in range(max_iter):
            # Fit model by updating mean of observed states
            for i in np.unique(s):
                mu[i] = np.mean(Y[s==i], axis=0)
            # Fit state sequence
            s_old = s.copy()
            loss_by_state = cdist(mu, Y, 'euclidean').T**2
            V = loss_by_state.copy()
            for t in range(n_obs-1, 0, -1):
                V[t-1] = loss_by_state[t-1] + (V[t] + Gamma).min(axis=1)
            s[0] = V[0].argmin()
            for t in range(1, n_obs):
                s[t] = (Gamma[s[t-1]] + V[t]).argmin()
            # Monitor convergence
            if len(np.unique(s)) == 1:
                break
            loss = min(V[0])
            if verbose:
                print('Iteration {}: {:.6e}'.format(it, loss))
            if tol:
                epsilon = loss_old - loss
                if epsilon < tol:
                    break
            elif np.array_equal(s, s_old):
                break
            loss_old = loss

        if (best_s is None) or (loss_old < best_loss):
            best_loss = loss_old
            best_s = s.copy()
        s = init_states(Y, n_states)

    return best_s, best_loss/2

def probabilistic_jump(Y, n_states, order, disc_size, jump_penalty=1e-5,
                                 initial_states=None, max_iter=10, n_init=10,
                                 tol=None, verbose=False):
    # Fit jump model using framework of Bemporad et al. (2018)
    n_obs, n_features = Y.shape

    if initial_states is not None:
        s = initial_states.copy()
    else:
        s = init_states(Y, n_states).astype(float)[:, np.newaxis]

        s_new = np.zeros((n_obs,n_states))
        for i in range(n_obs):
            s_new[i,int(s[i])] = 1
        s = s_new

    choices = np.array(list(partitions(int(1/disc_size), n_states)))/int(1/disc_size)
    Gamma = jump_penalty * (cdist(choices,choices, 'cityblock')/2)**order

    best_loss = None
    best_s = None

    for init in range(n_init):
        mu = np.zeros((n_states, n_features))
        loss_old = 1e10
        for it in range(max_iter):
            # Fit model by updating mean of observed states
            for i in range(n_states):
                if s.sum(axis = 0)[i] != 0:
                    mu[i] = np.sum(Y*s[:,i][:, np.newaxis], axis = 0)/sum(s[:,i])

            # Fit state sequence
            s_old = s.copy()
            loss_by_state = cdist(mu, Y, 'euclidean').T**2
            V = [None] * n_obs
            V[-1] = np.matmul(choices, loss_by_state[-1])

            for t in range(n_obs-1, 0, -1):
                V[t-1] = np.matmul(choices, loss_by_state[t-1])  + (V[t] + Gamma).min(axis=1)

            s[0] = choices[V[0].argmin()]
            for t in range(1, n_obs):
                s[t] = choices[(Gamma[np.where(np.all(choices == s[t-1],axis=1))] + V[t]).argmin()]
            # Monitor convergence

            if n_states - np.count_nonzero(s.sum(axis = 0) == 0) == 1:
                break

            loss = min(V[0])
            if verbose:
                print('Iteration {}: {:.6e}'.format(it, loss))
            if tol:
                epsilon = loss_old - loss
                if epsilon < tol:
                    break
            elif np.array_equal(s, s_old):
                break
            loss_old = loss

        if (best_s is None) or (loss_old < best_loss):
            best_loss = loss_old
            best_s = s.copy()

        s = init_states(Y, n_states).astype(float)[:, np.newaxis]
        s_new = np.zeros((n_obs,n_states))
        for i in range(n_obs):
            s_new[i,int(s[i])] = 1
        s = s_new

    return best_s, best_loss/2

def init_states(Y, n_states):
    # Generate initial states using K-means++ (Arthur and Vassilvitskii, 2007)
    n_obs, n_features = Y.shape
    centers = np.zeros((n_states, n_features))
    center_idx = np.random.randint(n_obs)
    centers[0] = Y[center_idx]
    n_local_trials = 2 + int(np.log(n_states))
    closest_dist_sq = cdist(centers[0, None], Y, 'euclidean')**2
    current_pot = closest_dist_sq.sum()

    for i in range(1, n_states):
        rand_vals = np.random.sample(n_local_trials) * current_pot
        candidate_ids = np.searchsorted(np.cumsum(closest_dist_sq),
                                        rand_vals)
        distance_to_candidates = cdist(Y[candidate_ids], Y, 'euclidean')**2
        # Decide which candidate is the best
        best_candidate = None
        best_pot = None
        best_dist_sq = None
        for trial in range(n_local_trials):
            # Compute potential when including center candidate
            new_dist_sq = np.minimum(closest_dist_sq,
                                     distance_to_candidates[trial])
            new_pot = new_dist_sq.sum()

            # Store result if it is the best local trial so far
            if (best_candidate is None) or (new_pot < best_pot):
                best_candidate = candidate_ids[trial]
                best_pot = new_pot
                best_dist_sq = new_dist_sq

        centers[i] = Y[best_candidate]
        current_pot = best_pot
        closest_dist_sq = best_dist_sq

    # Compute the state assignment
    states = cdist(centers, Y, 'euclidean').argmin(axis=0)

    return states

def get_weights(Y, states, max_features, n_states):
    # Find weights given a state sequence by maximizing the interstate distance
    BCSS = get_BCSS(Y, states)
    delta = binary_search(BCSS, max_features)
    w = calc_new_feature_weights(BCSS, delta)

    return w

def get_probabilistic_weights(Y, states, max_features, n_states):
    # Find weights given a state sequence by maximizing the interstate distance
    BCSS = get_probabilistic_BCSS(Y, states)
    delta = binary_search(BCSS, max_features)
    w = calc_new_feature_weights(BCSS, delta)

    return w

def get_BCSS(Y, states):
    # Find BCSS given a state sequence
    WCSS = np.zeros(Y.shape[1])
    for i in np.unique(states):
        mask = (states == i)
        if mask.sum() > 1:
            WCSS += np.square(Y[mask] - np.mean(Y[mask], axis=0)).sum(axis=0)
    TSS = np.square(Y - np.mean(Y, axis=0)).sum(axis=0)

    return TSS - WCSS

def get_probabilistic_BCSS(Y, states):
    # Find BCSS given a state sequence
    WCSS = np.zeros(Y.shape[1])

    mu = np.zeros((states.shape[1], Y.shape[1]))

    for i in range(states.shape[1]):
        mu[i] = np.sum(Y*states[:,i][:, np.newaxis], axis = 0)/sum(states[:,i])

    state_losses = np.zeros((states.shape[1], Y.shape[0], Y.shape[1]))

    for i in range(states.shape[1]):
        state_losses[i,:,:] = np.square(Y - mu[i])

    total_losses = np.zeros((Y.shape[0], Y.shape[1]))

    for i in range(states.shape[1]):
        total_losses += states[:,i][:, np.newaxis] * state_losses[i,:,:]

    WCSS = total_losses.sum(axis=0)

    TSS = np.square(Y - np.mean(Y, axis=0)).sum(axis=0)

    return TSS - WCSS

def binary_search(objective, norm_constraint, max_iter=15):
    l2n_arg = np.linalg.norm(objective)
    if l2n_arg == 0 or abs(objective / l2n_arg).sum() <= norm_constraint:
        return 0
    lam1 = 0
    lam2 = abs(objective).max() - 1e-5
    for iter in range(max_iter):
        su = soft_threshold(objective, (lam1 + lam2) / 2)
        if abs(su / np.linalg.norm(su)).sum() < norm_constraint:
            lam2 = (lam1 + lam2) / 2
        else:
            lam1 = (lam1 + lam2) / 2
        if (lam2 - lam1) < 1e-4:
            break

    return (lam1 + lam2) / 2

def calc_new_feature_weights(objective, delta):
    # Calculate feature weights using soft thresholding
    soft = soft_threshold(objective, delta)
    w = soft / np.linalg.norm(soft)

    return w

def soft_threshold(x, delta):

    return np.sign(x) * np.maximum(0, np.abs(x) - delta)

def partitions(n, b):
    masks = np.identity(b, dtype=int)
    for c in itertools.combinations_with_replacement(masks, n):
        yield sum(c)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('login', '0002_usuario_nome'),
    ]

    operations = [
        migrations.RenameField(
            model_name='usuario',
            old_name='nome',
            new_name='username',
        ),
    ]


import sys
input = sys.stdin.readline

def dfs(x, y):
    stack = []
    stack.append((x, y))

    while stack:
        for i in range(8):
            nx = x + dx[i]
            ny = y + dy[i]
            if 0<=nx<h and 0<=ny<w and arr[nx][ny] == '1':
                stack.append((x, y))
                arr[nx][ny] = '0'
                x = nx
                y = ny
                break
        else:
            x, y = stack.pop()

    return 1

dx = [-1, -1, -1, 0, 0, 1, 1, 1]
dy = [-1, 0, 1, -1, 1, -1, 0, 1]

in_, put_ = map(int, input().rstrip().split())

while in_ != 0 and put_ != 0:
    w, h = in_, put_
    arr = [input().rstrip().split() for _ in range(h)]
    ans = 0

    for x in range(h):
        for y in range(w):
            if arr[x][y] == '1':
                ans += dfs(x, y)
                arr[x][y] == '0'

    print(ans)
    in_, put_ = map(int, input().rstrip().split())


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from django.views.decorators.cache import never_cache

from helfertool.utils import nopermission
from registration.permissions import has_access, ACCESS_CORONA_VIEW, ACCESS_CORONA_EDIT
from registration.utils import get_or_404

from ..forms import ContactTracingDataForm
from ..models import ContactTracingData
from .utils import notactive

import logging

logger = logging.getLogger("helfertool.corona")

@login_required
@never_cache
def view_helper(request, event_url_name, helper_pk):
    event, job, shift, helper = get_or_404(event_url_name, helper_pk=helper_pk)

    # check permissions
    if not has_access(request.user, helper, ACCESS_CORONA_VIEW):
        return nopermission(request)

    # check if corona contact tracing is active
    if not event.corona:
        return notactive(request)

    # get data if it exists
    try:
        data = helper.contacttracingdata
    except ContactTracingData.DoesNotExist:
        data = None

    # render page
    context = {"event": event, "helper": helper, "data": data}
    return render(request, "corona/view_helper.html", context)

@login_required
@never_cache
def edit_helper(request, event_url_name, helper_pk):
    event, job, shift, helper = get_or_404(event_url_name, helper_pk=helper_pk)

    # check permissions
    if not has_access(request.user, helper, ACCESS_CORONA_EDIT):
        return nopermission(request)

    # check if corona contact tracing is active
    if not event.corona:
        return notactive(request)

    # get data if it exists
    try:
        data = helper.contacttracingdata
    except ContactTracingData.DoesNotExist:
        data = None

    # form
    form = ContactTracingDataForm(request.POST or None, instance=data, event=event)
    if form.is_valid():
        form.save(helper=helper)

        logger.info(
            "helper coronadata",
            extra={
                "user": request.user,
                "event": event,
                "helper": helper,
            },
        )

        return redirect("corona:view_helper", event_url_name=event_url_name, helper_pk=helper.pk)

    # render page
    context = {"event": event, "helper": helper, "form": form}
    return render(request, "corona/edit_helper.html", context)


from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.views.generic.simple import direct_to_template
from django.template import RequestContext
from images.models import Image
from images.forms import ImageForm

def images_add(request):
    if not request.user.is_authenticated():
        return HttpResponseRedirect('/staff/login/')
    elif request.user.has_perm('images.add_image'):
        if request.method == 'POST':
            form = ImageForm(request.POST, request.FILES)
            if form.is_valid():
                # save new image
                new_image = Image(name=form.cleaned_data['name'])
                if form.cleaned_data.get('caption'):
                    new_image.caption = form.cleaned_data['caption']
                if form.cleaned_data.get('credit'):
                    new_image.credit = form.cleaned_data['credit']
                new_image.image = form.cleaned_data['image']
                new_image.save()
                return HttpResponseRedirect(reverse('images.views.images_add_to_markup', args=[new_image.id]))
            else:
                return render_to_response('images/widget_add.html',
                    {'form': form},
                    context_instance=RequestContext(request))
        else:
            form = ImageForm()
            return render_to_response('images/widget_add.html',
                {'form': form},
                context_instance=RequestContext(request))
    else:
        return render_to_response('staff/access_denied.html',
            {'missing': 'add photos to',
                'staffapp': 'this entry'},
            context_instance=RequestContext(request))

##		  return create_object(request,
##							   model=Image,
##							   template_name='images/widget_addd.html',
##							   post_save_redirect='/images/widget/markup/%(id)i/')

def images_list(request):
    return direct_to_template(request, template='images/widget_add.html')

def images_add_to_markup(request, image_id):
    c = {'image': Image.objects.get(pk=image_id)}
    return direct_to_template(request, template='images/widget_add_to_markup.html', extra_context=c)


# Standard Library
import argparse
import os

class EnvDefault(argparse.Action):
    """
    Helper for the CLI argparse to allow setting defaults through environment variables
    Usage: In an argparse argument, set the Action to this class.
        Add the extra variable envvar added that has the name of the environment variable containing the default value.
    Example: parser.add_argument("--gitleaks-path", action=EnvDefault, envvar="RESC_GITLEAKS_PATH")
        This would result in the parser reading the env var if it exists and using it as the default,
        always to be overrideable using the cli argument.
    """
    def __init__(self, envvar, required=True, default=None, **kwargs):
        if not default and envvar:
            if envvar in os.environ:
                default = os.environ[envvar]
        if required and default:
            required = False
        super().__init__(default=default, required=required, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)


"""
called as optimal_model_search.py $TMPDIR $PACKAGEDIR $NPROC $PATTERNDIR $OUTDIR
"""
import sys
import itertools
import logging
import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq

from pathlib import Path
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import brier_score_loss
from multiprocessing import Pool

TMPDIR = Path(sys.argv[1])
PACKAGEDIR = sys.argv[2]
NPROC = int(sys.argv[3])
PATTERNDIR = Path(sys.argv[4])
OUTDIR = Path(sys.argv[5])

sys.path.append(PACKAGEDIR)
from Weave.models import permute_importance, compute_shaps, map_foldindex_to_groupedorder, HybridExceedenceModel

logging.basicConfig(filename= TMPDIR / 'permimp_train_q06.log', filemode='w', level=logging.DEBUG, format='%(process)d-%(relativeCreated)d-%(message)s')

path_complete = PATTERNDIR / 'precursor.multiagg.parquet'
path_y = PATTERNDIR / 'response.multiagg.trended.parquet'

def read_data(responseagg = 3, separation = -7, quantile: float = 0.8):
    """
    Returns the selcted X and y data
    A dataframe and a trended Series
    """
    y = pd.read_parquet(path_y).loc[:,(slice(None),responseagg,slice(None))].iloc[:,0] # Only summer
    X = pd.read_parquet(path_complete).loc[y.index,(slice(None),slice(None),slice(None),slice(None),separation,slice(None),slice(None))].dropna(axis = 0, how = 'any') # both metrics
    y = y.reindex(X.index)
    y = y > y.quantile(quantile)
    logging.debug(f'read y from {path_y} at resptimeagg {responseagg} trended, exceeding quantile {quantile}, and read dimreduced X from {path_complete} at separation {separation}')
    map_foldindex_to_groupedorder(X = X, n_folds = 5)
    logging.debug('restored fold order on dimreduced X')
    return X, y

def execute_shap(respseptup):
    """
    FUnction to fit model and call shapley values computation with certain arguments
    Can be paralellized
    """
    responseagg, separation = respseptup
    retpath = OUTDIR / str(responseagg) / str(separation)
    if not retpath.exists():
        X,y = read_data(responseagg = responseagg, separation = separation, quantile = 0.666)

        model = HybridExceedenceModel(fit_base_to_all_cv = True, max_depth = 5, n_estimators = 2500, min_samples_split = 30, max_features = 35, n_jobs = njobs_per_imp)
        shappies = compute_shaps(model, X, y, on_validation = False, use_background = True, bg_from_training = True, sample_background = 'standard', n_folds = 5, shap_kwargs = dict(check_additivity = False))
        retpath.mkdir(parents = True)
        pq.write_table(pa.Table.from_pandas(shappies), retpath / 'responsagg_separation.parquet')
        logging.debug(f'subprocess has written out SHAP frame at {retpath}')
    else:
        logging.debug(f'SHAP frame at {retpath} already exists')

def execute_perm_imp(respseptup):
    """
    Standalone function writes the importance of this combination
    of responseagg and separation to a subdirectory
    no returns
    """
    responseagg, separation = respseptup
    retpath = OUTDIR / str(responseagg) / str(separation)
    if not retpath.exists():
        X,y = read_data(responseagg = responseagg, separation = separation, quantile = 0.8)
        #def wrapper(self, *args, **kwargs):
        #    return self.predict_proba(*args,**kwargs)[:,-1] # Last class is True
        #RandomForestClassifier.predict = wrapper # To avoid things inside permutation importance package  where it is only possible to invoke probabilistic prediction with twoclass y.
        #m = RandomForestClassifier(max_depth = 7, n_estimators = 1500, min_samples_split = 40, max_features = 35, n_jobs = njobs_per_imp)
        model = HybridExceedenceModel(fit_base_to_all_cv = True, max_depth = 5, n_estimators = 2500, min_samples_split = 30, max_features = 35, n_jobs = njobs_per_imp)
        ret = permute_importance(model, X_in = X, y_in = y, on_validation = False, evaluation_fn = brier_score_loss, n_folds = 5, perm_imp_kwargs = dict(nimportant_vars = 30, njobs = njobs_per_imp, nbootstrap = 1500))
        retpath.mkdir(parents = True)
        pq.write_table(pa.Table.from_pandas(ret), retpath / 'responsagg_separation.parquet')
        logging.debug(f'subprocess has written out importance frame at {retpath}')
    else:
        logging.debug(f'importance frame at {retpath} already exists')

if __name__ == "__main__":
    """
    Parallelized with multiprocessing over repsagg / separation models
    """
    #njobs_per_imp = 1
    #nprocs = NPROC // njobs_per_imp
    #logging.debug(f'Spinning up {nprocs} processes with each {njobs_per_imp} for shapley')
    #responseaggs = np.unique(pd.read_parquet(path_y).columns.get_level_values('timeagg'))
    #separations = np.unique(pd.read_parquet(path_complete).columns.get_level_values('separation'))
    #with Pool(nprocs) as p:
    #    p.map(execute_shap, itertools.product(responseaggs, separations))
    """
    Parallelized with threading for forest fitting and permutation importance per respagg / separation model
    """
    #responseaggs = np.unique(pd.read_parquet(path_y).columns.get_level_values('timeagg'))
    #separations = np.unique(pd.read_parquet(path_complete).columns.get_level_values('separation'))
    #njobs_per_imp = NPROC
    #for respagg_sep in itertools.product(responseaggs, separations):
    #    execute_perm_imp(respagg_sep)
    njobs_per_imp = 1
    nprocs = NPROC // njobs_per_imp
    logging.debug(f'Spinning up {nprocs} processes with each {njobs_per_imp} for permimp')
    responseaggs = np.unique(pd.read_parquet(path_y).columns.get_level_values('timeagg'))
    separations = np.unique(pd.read_parquet(path_complete).columns.get_level_values('separation'))
    with Pool(nprocs) as p:
        p.map(execute_perm_imp, itertools.product(responseaggs, separations))
    #njobs_per_imp = NPROC
    #execute_perm_imp((31,-15))


import pandas as pd
from sklearn.linear_model import LinearRegression

'''dealing with the data'''
def preprocessing(country):
    rows = data.loc[(data['countriesAndTerritories'] == country)]
    x = rows['dateRep'].iloc[::-1].reset_index(drop=True)
    for i in range(x.size):
        x[i] = i
    size = x.size
    secondx=[]
    for j in range (size):
        secondx.insert(j, [j])
    y = rows['cases'].iloc[::-1].reset_index(drop=True)
    secondy=[]
    for s in range (size):
        secondy.insert(s, y[s])
    return secondx,secondy,size

'''train the model'''
def train(x,y,size):
    x_train = x[int(size*0.3):]
    y_train = y[int(size*0.3):]
    #build model
    model = LinearRegression()
    model.fit(x_train, y_train)
    predictions = model.predict([[size], [size+1], [size+2], [size+3], [size+4], [size+5], [size+6]])
    for i in range(7):
        if predictions[i]<1:
            predictions[i]=0
        predictions[i] = int(predictions[i])
    return predictions

def main():
    countrylist = data.countriesAndTerritories.unique().tolist()
    l=[]
    for k in range(len(countrylist)):
        x,y, size = preprocessing(countrylist[k])
        predictions = train(x,y, size)
        l.append(predictions)
    dat = ['10/9', '10/10', '10/11', '10/12', '10/13', '10/14', '10/15']
    result = pd.DataFrame(zip(*l), columns = countrylist, index=dat)
    result.to_csv('StudentID_HW1.csv')

'''file import'''
dataset_file_path = "your/dataset/path/dataset.csv"
data = pd.read_csv(dataset_file_path)
main()


import sys
from catchException import exception_handler
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from requests import ReadTimeout, ConnectTimeout, HTTPError, Timeout, ConnectionError
import pandas as pd
import os
# import requests

# import urllib3

## handle exception properly
## make it summarised info , hide traceback
sys.excepthook = exception_handler
#####

def collectApiData(session=None, apitype="surveys", survey_id = 1, limit=40000, offset=0, token=" " ):
    '''
    function takes 6 parameters and returns the data from the api.
    session : request session. this can be reused for the different api type
    apitype:  the type of data to collect. can be answers or surveys. This selection determines the api to be used
    survey_id: each survey has survey id. this is the corresponing survey id . Get the survey id from the surveys api
    limit: the limit of the data to be collected
    offset: the offset where to start next data collection from
    token: the token to be used in the api call

    '''

    ###########################################################################
    #### check if sapi type correct
    if  apitype == "surveys":
        api = "https://endapi.truefeedback.io/dataplatform/survey/"
    elif apitype == "answers":
        api = 'https://endapi.truefeedback.io/dataplatform/survey/'  + str(survey_id)  + '/'  + 'answers?' + 'limit='+ str(limit) + '&offset=' + str(offset)
    else:
       raise Exception("wrong api type. The api type is a string that can only be answers or surveys")

    ###########################################################################

    ######## inform user of what is happening : that is connection to api is being made ########
    print("The data api endpoint is: ", api)
    ###########################################################################

    ###########################################################################
    ### try catch statement for any error that occurs during connection
    try:

        ##
        print("connecting to data api endpoint ...")
        ## configure session retry
        retries = Retry(connect=4,total=3,
                        backoff_factor=0.1,
                        status_forcelist=[ 500, 502, 503, 504 ]) ## force status retry if code returned is 500, 502, 504

        ### mount an adapter
        # "pool_maxsize" is the number of connections to keep around per host
        # (this is useful for multi-threaded applications), whereas "pool_connections" is the number of host-pools
        # to keep around. For example, if you're connecting to 100 different hosts, and pool_connections=10, then only the latest 10 hosts'
        # connections will be re-used.
        session.mount('http://', HTTPAdapter(max_retries=retries,pool_connections=10, pool_maxsize=10))
        ##

        ### establish a get on the api
        getApi = session.get(api, headers={"auth": token} )

        ## the content of the extracted data
        a = getApi.content
        ## read extracted data into pandas
        df=pd.read_json(a)
        print("my df: ", df.head())
        return df
    ###########################################################################

    ###########################################################################
    ## from here is the exception statement for try
    #### connection related error
    except  (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError) as e:
        raise Exception('There is an http connection problem. Check the connection configuration', str(e))
    ### value related error
    except ValueError as e:
        raise ValueError('Data not in json format or did you supply wrong token?.', str(e))
    ### catch  keyboard interrupt
    except KeyboardInterrupt as e:
        print(" \t You interrupted the program.")
        try:
            sys.exit(0)
        except:
            os._exit(0)
    ## if exception not caught from the previous exceptions
    except:
        raise Exception('a problem has occurred with connection/connection parameters or data format or mising library import or non initialised session')
        #sys.exit()
    ##########################################################################


from pprint import pprint
from functools import reduce
from collections import defaultdict

data = [l.strip() for l in open("input.txt","r").readlines()]
pprint(data)

# part 1
# vowels = ["a","e","i","o","u"]
# forbidden = ["ab", "cd", "pq", "xy"]
# nice = 0
# for line in data:
#     vows = sum([1 for ch in line if ch in vowels])
#     double = sum([1 for i in range(len(line)-1) if line[i] == line[i+1]])
#     wrong = sum([1 for i in range(len(line)-1) if "{}{}".format(line[i],line[i+1]) in forbidden])
#     if vows >= 3 and double > 0 and wrong == 0:
#         nice += 1
# print(nice)

nice = 0
for line in data:
    pairs = defaultdict(lambda: set())
    for i in range(0,len(line)-1,1): pairs["{}{}".format(line[i],line[i+1])].update((i,i+1))
    right = sum([1 for p in pairs if (len(pairs[p])/2) >= 2])
    gapped = sum([1 for i in range(len(line)-2) if line[i] == line[i+2]])
    if right >= 1 and gapped >= 1:
        nice += 1
print(nice)