# WebDriver 对象有好几种方法，用于在页面中寻找元素。它们被分成find_element_*和
# find_elements_*方法。find_element_*方法返回一个 WebElement 对象，代表页面中
# 匹配查询的第一个元素。find_elements_*方法返回 WebElement_*对象的列表，包含
# 页面中所有匹配的元素。
#
# selenium 的 WebDriver 方法，用于寻找元素
# 方法名                                                 返回的 WebElement 对象/列表
# browser.find_element_by_class_name(name)              使用 CSS 类 name 的元素
# browser.find_elements_by_class_name(name)
#
# browser.find_element_by_css_selector(selector)        匹配 CSS selector 的元素
# browser.find_elements_by_css_selector(selector)
#
# browser.find_element_by_id(id)                        匹配 id 属性值的元素
# browser.find_elements_by_id(id)
#
# browser.find_element_by_link_text(text)               完全匹配提供的 text 的元素
# browser.find_elements_by_link_text(text)
#
# browser.find_element_by_partial_link_text(text)       包含提供的 text 的元素
# browser.find_elements_by_partial_link_text(text)
#
# browser.find_element_by_name(name)                    匹配 name 属性值的元素
# browser.find_elements_by_name(name)
#
# browser.find_element_by_tag_name(name)                匹配标签 name 的元素
# browser.find_elements_by_tag_name(name)               (大小写无关，元素匹配'a'和'A')
#
#
# 除了*_by_tag_name()方法，所有方法的参数都是区分大小写的。如果页面上没
# 有元素匹配该方法要查找的元素，selenium 模块就会抛出 NoSuchElement 异常。如
# 果你不希望这个异常让程序崩溃，就在代码中添加 try 和 except 语句。
# 一旦有了 WebElement 对象，就可以其中的属性，或调用其中的方法
#
# WebElement 的属性和方法
# 属性或方法                                描述
# tag_name                                标签名，例如 'a'表示元素
# get_attribute(name)                     该元素 name 属性的值
# text                                    该元素内的文本，例如hello中的'hello'
# clear()                                 对于文本字段或文本区域元素，清除其中输入的文本
# is_displayed()                          如果该元素可见，返回 True，否则返回 False
# is_enabled()                            对于输入元素，如果该元素启用，返回 True，否则返回 False
# is_selected()                           对于复选框或单选框元素，如果该元素被选中，选择 True，否则返回 False
# location                                一个字典，包含键'x'和'y'，表示该元素在页面上的位置

from selenium import webdriver

chrome_drive = r"/home/ryan/Downloads/chromedriver"
browser = webdriver.Chrome(executable_path=chrome_drive)
browser.get("https://www.baidu.com")

try:
    ret = browser.find_element_by_class_name('title-content-title')
    print(f"Find the {ret.tag_name} element with that class name!")
except:
    print("Fail to find the a element with that class name")


from relay_sdk import Interface, WebhookServer
from quart import Quart, request, jsonify, make_response

import logging
import json

relay = Interface()
app = Quart('bitbucket-event')

logging.getLogger().setLevel(logging.INFO)

@app.route('/', methods=['POST'])
async def handler():
    bitbucket_event = request.headers.get('X-Event-Key')

    if bitbucket_event is None:
        return {'message': 'not a valid Bitbucket event'}, 400, {}
    if bitbucket_event == 'ping':
        return {'message': 'success'}, 200, {}

    logging.info("Received event from Bitbucket: {}".format(bitbucket_event))

    event_payload = await request.get_json()
    logging.info("Received the following webhook payload: \n%s", json.dumps(event_payload, indent=4))

    if event_payload is None:
        return {'message': 'not a valid Bitbucket event'}, 400, {}

    relay.events.emit({
          'event_payload': event_payload,
          'bitbucket_event': bitbucket_event
      })

    return {'message': 'success'}, 200, {}

if __name__ == '__main__':
    WebhookServer(app).serve_forever()


class Microphone(QtMultimedia.QAudioRecorder):
    """Attempt to use qt multimedia to feed listener

    mic = Microphone()
    # mic.choose_default()
    if mic.default_pipe():
        mic.record()

    """

    def default_format(self):
        self.setAudioCodec('audio/x-raw')
        settings = QtMultimedia.QAudioEncoderSettings()
        settings.setChannelCount(1)
        settings.setSampleRate(16000)
        self.setEncodingSettings(settings)

    def default_pipe(self):
        path = '/run/user/%s/listener/audio' % os.geteuid()
        if os.path.exists(path):
            self.setOutputLocation(QtCore.QUrl.fromLocalFile(path))
            return True
        else:
            log.warning("Listener is not currently listening")


#!/usr/bin/env python
# -*- coding: ascii -*-
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
"""Test suite for LCD routines"""

import os
import sys

try:
    if sys.version_info < (2, 3):
        # unittest2 does NOT work under Python 2.2.
        raise ImportError
    import unittest2
    unittest = unittest2
except ImportError:
    import unittest
    unittest2 = None

import pylcdsysinfo

IMAGE_DIR = os.path.dirname(os.path.abspath(__file__))
IMAGE_DIR = os.path.join(IMAGE_DIR, 'images')
IMAGE_320X240_DIR = os.path.join(IMAGE_DIR, '320x240')
ICON_DIR = os.path.join(IMAGE_DIR, 'icons')

class ImageTests(unittest.TestCase):

    def skip(self, reason):
        """Skip current test because of `reason`.

        NOTE expects unittest2, and defaults to "pass" if not available.
        """
        if unittest2:
            raise unittest2.SkipTest(reason)

    def testimage_pattern_320x240_rgb565_bmp(self):
        filename = 'testpattern_rgb565.bmp'
        filename = os.path.join(IMAGE_320X240_DIR, filename)
        canon_filename = os.path.splitext(filename)[0] + '.raw'

        f = open(filename, 'rb')
        data = f.read()
        f.close()
        raw_data = pylcdsysinfo._bmp_to_raw(data)

        f = open(canon_filename, 'rb')
        canon_data = f.read()
        f.close()
        self.assertEqual(canon_data, raw_data, 'does not match canon')

    def testimage_pattern_320x240_rgb888_png(self):
        filename = 'testpattern.png'
        """NOTE the rgb888 -> rgb565  conversion implemented in ffmeg is
        slightly different to the technique applied in image_to_raw().
        This is why a different raw file is used, also testpattern_rgb565.bmp
        is already in rgb565 format, the png is full 16/24 bit depth.
        """
        filename = os.path.join(IMAGE_320X240_DIR, filename)
        canon_filename = os.path.splitext(filename)[0] + '.raw'

        im = pylcdsysinfo.Image.open(filename)
        raw_data = pylcdsysinfo.image_to_raw(im)

        f = open(canon_filename, 'rb')
        canon_data = f.read()
        f.close()
        self.assertEqual(canon_data, raw_data, 'does not match canon')

if __name__ == '__main__':
    unittest.main()


import numpy as np
from math import sqrt
from collections import Counter

class KNNClassifier:
    def __init__(self, k):
        """ 初始化kNN分类器 """
        assert k >= 1, " k must be valid "
        self.k = k
        self._X_train = None
        self._y_train = None

    def fit(self, X_train, y_train):
        """ 根据参数训练模型 """
        assert X_train.shape[0] == y_train.shape[0], \
            "the size of X_train must be equal to the size of y_trian"
        assert self.k <= X_train.shape[0], \
            "the size of X_train must be as least k"
        self._X_train = X_train
        self._y_train = y_train
        return self

    def predict(self,X_predict):
        """ 预测 """
        assert self._X_train is not None and self._y_train is not None, \
            "must fit before predict"
        assert X_predict.shape[1] == self._X_train.shape[1], \
            "the feature number of X_predict must be equal to X_train"
        y_predict = [self._predict(x_predict) for x_predict in X_predict]
        return np.array(y_predict)

    def _predict(self, x_predict):
        """ 给单个预测数据x_predict,返回x的预测结果 """
        assert x_predict.shape[0] == self._X_train.shape[1], \
            "the feature number of x must be equal to X_train"
        # kNN过程
        diatances = [sqrt(np.sum((x_train - x_predict)**2)) for x_train in self._X_train]
        nearest = np.argsort(diatances)
        votes = Counter(self._y_train[i] for i in nearest[:self.k])
        return votes.most_common(1)[0][0]

    def __repr__(self):
        return "KNN(k=%d)" % self.k


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 10 16:21:57 2017

@author: yipinp

The package will do the data set preparation
including:
      (1) crop the image size to fixed image size and support parameter setting(256x256)
      (2) only support RGB 3 channel or Gray  channel, others will convert to RGB or gray(to Image, ignore it)
      (3) support image rotation for better training set generation.(need to provide theta angle) and do scaler to fixed size
      (4) Add preprocessing step such as add noise/fog and so on.
      (5) normalization the input image range [0,255] to [0,1] for light independent.(ndarray)
      (6) generate patch sets based on (patch size, stride, number)

"""
# -*- coding: utf-8 -*-
import os
import os.path
import cv2
import numpy as np
from matplotlib import pyplot as plt
import random
import tensorflow as tf
import time
from skimage.measure import compare_ssim as ssim
from skimage.measure import compare_psnr as psnr
from sklearn import preprocessing

#from tflearn.layers.core import input_data,dropout,fully_connected
#from tflearn.layers.conv import conv2d,max_pool_2d
#from tflearn.layers.normalization import local_response_normalization

"""
    Scan one directory and find all files in the directory. return file list
"""
def scan_image_directories(directory_path):
    result = []
    for root,dirs,files in os.walk(directory_path):
        for filename in files:
            print(os.path.join(root,filename))
            #check filename for jpg,jpeg,png only
            if(os.path.splitext(filename)[1] != ".jpg" and os.path.splitext(filename)[1] != ".jpeg" and os.path.splitext(filename)[1] != ".png"):
                continue
            result.append(os.path.join(root,filename))
    return result

def random_image_list(filelist,seed):
    random.seed(seed)
    new_file_list =[]
    for n in random.sample(range(len(filelist)),len(filelist)):
        new_file_list.append(filelist[n])

    return new_file_list

"""
        Add gaussian noise or salt peper noise
"""
def GaussianWhiteNoiseForRGB(imgIn,dst_size,mean,sigma):
    img = np.zeros(imgIn.shape)
    gray = 255
    zu = []
    zv = []
    for i in range(0,dst_size[0]):
        for j in range(0,dst_size[1],2):
            r1 = np.random.random_sample()
            r2 = np.random.random_sample()
            z1 = mean + sigma*np.cos(2*np.pi*r2)*np.sqrt((-2)*np.log(r1))
            z2 = mean + sigma*np.sin(2*np.pi*r2)*np.sqrt((-2)*np.log(r1))
            zu.append(z1)
            zv.append(z2)
            img[i,j,0] = np.clip(int(imgIn[i,j,0] + z1),0,gray)
            img[i,j+1,0] = np.clip(int(imgIn[i,j+1,0] + z2),0,gray)
            img[i,j,1] = np.clip(int(imgIn[i,j,1] + z1),0,gray)
            img[i,j+1,1] = np.clip(int(imgIn[i,j+1,1] + z2),0,gray)
            img[i,j,2] = np.clip(int(imgIn[i,j,2] + z1),0,gray)
            img[i,j+1,2] = np.clip(int(imgIn[i,j+1,2] + z2),0,gray)
    return img

def saltAndPepperForRGB(img,percetage):
        percetage = salt_percent
        dst_size = image_size
        width = dst_size[1]
        height = dst_size[0]
        NoiseNum = int(width*height*percetage)
        for i in range(NoiseNum):
            randx    = np.random.randint(0,width-1)
            randy    = np.random.randint(0,height-1)
            #print img.shape,randx,randy
            if np.random.randint(0,1):
                img[randy,randx,0] =  0
                img[randy,randx,1] =  0
                img[randy,randx,2] =  0
            else :
                img[randy,randx,0] =  255
                img[randy,randx,1] =  255
                img[randy,randx,2] =  255

        return img

def GaussianWhiteNoiseForGray(imgIn,dst_size,mean,sigma):
    img = np.zeros(imgIn.shape)
    gray = 255
    zu = []
    zv = []
    for i in range(0,dst_size[0]):
        for j in range(0,dst_size[1],2):
            r1 = np.random.random_sample()
            r2 = np.random.random_sample()
            z1 = mean + sigma*np.cos(2*np.pi*r2)*np.sqrt((-2)*np.log(r1))
            z2 = mean + sigma*np.sin(2*np.pi*r2)*np.sqrt((-2)*np.log(r1))
            zu.append(z1)
            zv.append(z2)
            img[i,j] = np.clip(int(imgIn[i,j] + z1),0,gray)
            img[i,j+1] = np.clip(int(imgIn[i,j+1] + z2),0,gray)
            """
            img[i,j,1] = np.clip(int(img[i,j] + z1),0,gray)
            img[i,j+1,1] = np.clip(int(img[i,j+1] + z2),0,gray)
            img[i,j,2] = np.clip(int(img[i,j] + z1),0,gray)
            img[i,j+1,2] = np.clip(int(img[i,j+1] + z2),0,gray)
            """
    return img

def saltAndPepperForRGB(img,percetage):
        percetage = salt_percent
        dst_size = image_size
        width = dst_size[1]
        height = dst_size[0]
        NoiseNum = int(width*height*percetage)
        for i in range(NoiseNum):
            randx    = np.random.randint(0,width-1)
            randy    = np.random.randint(0,height-1)
            #print img.shape,randx,randy
            if np.random.randint(0,1):
                img[randy,randx,0] =  0
                img[randy,randx,1] =  0
                img[randy,randx,2] =  0
            else :
                img[randy,randx,0] =  255
                img[randy,randx,1] =  255
                img[randy,randx,2] =  255

        return img

"""
    read image to numpy matrix and Normalization range to [0,1],
    generate patch based on patch_size,stride,number

"""
def get_normal_param(imageIn,mode):
    if mode == 0 :
        mean = np.mean(imageIn.reshape(-1))
        stddev = np.std(imageIn.reshape(-1))
        return mean,stddev
    else:
       dmin = np.min(imageIn)
       dmax = np.max(imageIn)
       return dmin,dmax

def image_normalization(imgIn,p0,p1,mode):
   # return preprocessing.scale(imgIn)
    if mode == 0:
        #img_norm = (imgIn/255.0 - 0.5)*0.2;
        img_norm = (imgIn - p0)/p1
        #img_norm = imgIn/255.0
        return img_norm
    else:
        return (imgIn - p0)/(p1 - p0)

def image_renorm(frame,p0,p1,mode):
    if mode == 0:
        #frame = (frame*5.0 + 0.5)*255.0
        frame = frame * p1 + p0

    else:
        frame = frame * (p1 - p0) + p0

    frame = np.array(frame,dtype=int)
    frame = frame.clip(0,255)
    return frame

#horizontal scan first then vertical,output array is [patch_height,patch_width,channel,number]
def next_batch(result,batch_num):
    global patch_current_x
    global patch_current_y
    global current_file_id
    global current_image
    global current_image_true
    global mode
    image = current_image
    image_true = current_image_true
    patch_height = patch_size[0]
    patch_width  = patch_size[1]
    num = batch_num
    stride = patch_stride
    current_x = patch_current_x
    current_y = patch_current_y

    if current_file_id >= len(result):
        current_file_id = 0   #reload training set if finished
       # print("All training set is finished, need to reset and shuffle the training set now!")
        random.shuffle(result)

    if current_x == 0 and current_y == 0 and current_file_id < len(result):
        image,image_true,_,_,_ = get_one_image(result[current_file_id])
        current_image = image
        current_image_true = image_true
        current_file_id = current_file_id + 1

    c = np.zeros((num,patch_height,patch_width),dtype="float32")
    c_true = np.zeros((num,patch_height,patch_width),dtype="float32")
    for i in range(num):
        a = image[current_y:min(current_y+patch_height,image.shape[0]),current_x:min(current_x+patch_width,image.shape[1])]
        b = image_true[current_y:min(current_y+patch_height,image.shape[0]),current_x:min(current_x+patch_width,image.shape[1])]
        #Fill 0 when out of picture
        c[i,:a.shape[0],:a.shape[1]] = a
        c_true[i,:a.shape[0],:a.shape[1]] = b
        #update next patch coordination
        if current_x+stride >= image.shape[1]:
            current_x = 0
            current_y += stride
        else:
            current_x += stride

        if current_y + stride >= image.shape[0]:
            #current_y = image.shape[0] + 1
            current_x = 0
            current_y = 0
            break

    patch_current_x = current_x
    patch_current_y = current_y
    #print("get batch number:",i,num,current_file_id,len(result),patch_current_x,patch_current_y,image.shape)
    return np.reshape(c[0:i+1,:,:],(i+1,-1)),np.reshape(c_true[0:i+1,:,:],(i+1,-1))

def get_patches_one_image(image_name):
    global patch_current_x
    global patch_current_y
    global mode
    patch_height = patch_size[0]
    patch_width  = patch_size[1]
    stride = patch_stride
    image,image_true,_,p0,p1 = get_one_image(image_name)
    height_in_patch = (image.shape[0] + patch_stride - 1)//patch_stride
    width_in_patch = (image.shape[1] + patch_stride - 1)//patch_stride
    num = height_in_patch * width_in_patch
    c = np.zeros((num,patch_height,patch_width),dtype="float32")
    c_true = np.zeros((num,patch_height,patch_width),dtype="float32")
    current_x = patch_current_x
    current_y = patch_current_y

    for i in range(num):
        a = image[current_y:min(current_y+patch_height,image.shape[0]),current_x:min(current_x+patch_width,image.shape[1])]
        c[i,:a.shape[0],:a.shape[1]] = a
        a_true = image_true[current_y:min(current_y+patch_height,image.shape[0]),current_x:min(current_x+patch_width,image.shape[1])]
        c_true[i,:a_true.shape[0],:a_true.shape[1]] = a_true
        if current_x+patch_stride >= image.shape[1]:
            current_x = 0
            current_y += stride
        else:
            current_x += stride
    return np.reshape(c[0:i+1,:,:],(i+1,-1)),np.reshape(c_true[0:i+1,:,:],(i+1,-1)),p0,p1,num

"""
    convert image to fixed size, and do 3x3 matrix multiple
"""
def get_one_image(filename):
    dst_size = image_size
    img_origin = cv2.imread(filename)
    global channel
    global mode
    global theta
    #convert to gray image
    if channel == 1:
        img_origin = cv2.cvtColor(img_origin,cv2.COLOR_BGR2GRAY)
    img_true = cv2.resize(img_origin,dst_size,interpolation=cv2.INTER_CUBIC)
    rows,cols = img_true.shape
    M = cv2.getRotationMatrix2D((cols/2.0,rows/2.0),theta,1.0)
    img_true = cv2.warpAffine(img_true,M,(rows,cols))

    if flip_mode != None :
        img_true = cv2.flip(img_true,flip_mode)

    if noise_model == 1:
        img = GaussianWhiteNoiseForGray(img_true,dst_size,noise_mean,noise_sigma)
    elif noise_model == 2:
        img = saltAndPepperForRGB(img_true,0)
    else :
        img = img_true
    p0,p1 = get_normal_param(img,mode)
    img = image_normalization(img,p0,p1,mode)
    p0_true,p1_true = get_normal_param(img_true,mode)
    img_true1 = image_normalization(img_true, p0,p1,mode)
    return img,img_true1,img_true,p0,p1

    #"""
    plt.imshow(img)
    plt.show()
    print(img.shape)
    print(img_origin.shape)
    #"""

def get_golden_image_show(filename):
    dst_size = image_size
    img_origin = cv2.imread(filename)
    global channel
    #convert to gray image
    if channel == 1:
        img_origin = cv2.cvtColor(img_origin,cv2.COLOR_BGR2GRAY)

    img_true = cv2.resize(img_origin,dst_size,interpolation=cv2.INTER_CUBIC)
    rows,cols = img_true.shape
    M = cv2.getRotationMatrix2D((cols/2.0,rows/2.0),theta,1.0)
    img_true = cv2.warpAffine(img_true,M,(rows,cols))

    if flip_mode != None :
        img_true = cv2.flip(img_true,flip_mode)

    return img_true

#Horizontal patch scan
def image_recovery(frame_height,frame_width,patch_height,patch_width,patch_stride,patches):
    frame_width_in_patch = (frame_width + patch_stride - 1)//patch_stride
    frame_height_in_patch = (frame_height + patch_stride - 1)//patch_stride
    frame = np.ones((frame_height_in_patch*patch_height,frame_width_in_patch*patch_width)) * -1
    for i in range(frame_height_in_patch):
        for j in range(frame_width_in_patch):
            patch_x = j * patch_stride
            patch_y = i * patch_stride
          #  print("patch:",patch_x,patch_y,i,j,frame_height_in_patch,frame_width_in_patch,patch_stride,patches.shape,i*frame_width_in_patch+j)
            patch = patches[i*frame_width_in_patch+j,:]
            np.savetxt("patch.txt",patches,fmt="%f")

            for m in range(patch_height):
                for n in range(patch_width):

                    if frame[patch_y+m][patch_x+n] < 0:  #the first patch
                        if (patch_y+m < frame_height) and (patch_x+n < frame_width) :
                            frame[patch_y+m][patch_x+n] = patch[m*patch_width + n]
                    else :
                        if (patch_y+m < frame_height) and (patch_x+n < frame_width) :
                            frame[patch_y+m][patch_x+n] += patch[m*patch_width + n]
                            frame[patch_y+m][patch_x+n] /=2.0;

    np.savetxt("frame.txt",frame[0:frame_height,0:frame_width],fmt="%f")
    return frame[0:frame_height,0:frame_width]

"""
    image quality evalution psnr/ssim

    mse = ((image-image_golden)**2).mean()
    if mode == "PSNR":
        cost = 10*np.log10((255**2)/mse)
    elif mode == "SSIM":
        cost = ssim(image,image_golden)

"""
def evaluate_image(mode,image,image_golden):

    if mode == "PSNR":
        cost = psnr(image,image_golden,255.0)
    elif mode == "SSIM":
        cost = ssim(image,image_golden,data_range=255.0)

    return cost

"""
   ---------------------------------------------------------------------------
                 Basic layers & MLP & AlexNet
"""
# Create model
def multilayer_perceptron(x, weights, biases,activate):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    if activate =="Relu":
        layer_1 = tf.nn.relu(layer_1)
    elif activate =="Sigmoid":
        layer_1 = tf.sigmoid(layer_1)
    else:
        layer_1 = tf.tanh(layer_1)

    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    if activate =="Relu":
        layer_2 = tf.nn.relu(layer_2)
    elif activate =="Sigmoid":
        layer_2 = tf.sigmoid(layer_2)
    else:
        layer_2 = tf.tanh(layer_2)

    # Hidden layer with RELU activation
    layer_3 = tf.add(tf.matmul(layer_2, weights['h3']), biases['b3'])
    if activate =="Relu":
        layer_3 = tf.nn.relu(layer_3)
    elif activate =="Sigmoid":
        layer_3 = tf.sigmoid(layer_3)
    else:
        layer_3 = tf.tanh(layer_3)

    # Output layer with linear activation
    out_layer = tf.matmul(layer_3, weights['out']) + biases['out']

    return out_layer

# Create some wrappers for simplicity
def conv2d(x, W, b, strides=1):
    # Conv2D wrapper, with bias and relu activation
    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')
    x = tf.nn.bias_add(x, b)
    return tf.tanh(x)
    return tf.nn.relu(x)

def norm(name, l_input, lsize=4):
    return tf.nn.lrn(l_input, lsize, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name=name)

def conv2d_batch(x, W, b,phase,strides=1):
    # Conv2D wrapper, with bias and relu activation
    x = tf.nn.conv2d(x, W, strides=[1, strides,strides, 1], padding='SAME')
    x = tf.nn.bias_add(x, b)
    x = tf.contrib.layers.batch_norm(x,
                                     center=True, scale=True,
                                     is_training=phase
                                   )
    return tf.tanh(x)
    return tf.nn.relu(x)

def maxpool2d(x, k=2):
    # MaxPool2D wrapper
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],
                          padding='SAME')

def max_pool(name, l_input, k):
    return tf.nn.max_pool(l_input, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME', name=name)

# Create model
def conv_net(x, weights, biases, dropout):
    # Reshape input picture
    x = tf.reshape(x, shape=[-1, 28, 28, 1])

    # Convolution Layer
    conv1 = conv2d(x, weights['h1'], biases['b1'])
    # Max Pooling (down-sampling)
    conv1 = maxpool2d(conv1, k=2)

    # Convolution Layer
    conv2 = conv2d(conv1, weights['h2'], biases['b2'])
    # Max Pooling (down-sampling)
    conv2 = maxpool2d(conv2, k=2)

    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer input
    fc1 = tf.reshape(conv2, [-1, weights['h3'].get_shape().as_list()[0]])
    fc1 = tf.add(tf.matmul(fc1, weights['h3']), biases['b3'])
    fc1 = tf.nn.relu(fc1)
    # Apply Dropout
    fc1 = tf.nn.dropout(fc1, dropout)

    # Output, class prediction
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    return out

def conv_net_batch(x, weights, biases,phase):
    # Reshape input picture
    x = tf.reshape(x, shape=[-1, 28, 28, 1])

    # Convolution Layer
    conv1 = conv2d_batch(x, weights['h1'], biases['b1'],phase)
    # Max Pooling (down-sampling)
    conv1 = maxpool2d(conv1, k=2)

    # Convolution Layer
    conv2 = conv2d_batch(conv1, weights['h2'], biases['b2'],phase)
    # Max Pooling (down-sampling)
    conv2 = maxpool2d(conv2, k=2)

    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer input
    fc1 = tf.reshape(conv2, [-1, weights['h3'].get_shape().as_list()[0]])
    fc1 = tf.add(tf.matmul(fc1, weights['h3']), biases['b3'])

    fc1 = tf.contrib.layers.batch_norm(fc1,
                                       center=True, scale=True,
                                       is_training=phase
                                     )
    fc1 = tf.reshape(fc1, [-1, weights['out'].get_shape().as_list()[0]])
    #fc1 = tf.nn.relu(fc1)
    fc1 = tf.tanh(fc1)
    fc1 = tf.nn.dropout(fc1, dropout)
    # Output, class prediction
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    return out

def alex_net(_X, _weights, _biases, _dropout):
    _X = tf.reshape(_X, shape=[-1, 28, 28, 1])
    conv1 = conv2d(_X, _weights['wc1'], _biases['bc1'])
    pool1 = max_pool('pool1', conv1, k=2)
    # 归一化层
    norm1 = norm('norm1', pool1, lsize=4)
    # Dropout
    norm1 = tf.nn.dropout(norm1, _dropout)

    # 卷积
    conv2 = conv2d( norm1, _weights['wc2'], _biases['bc2'])
    # 下采样
    pool2 = max_pool('pool2', conv2, k=2)
    # 归一化
    norm2 = norm('norm2', pool2, lsize=4)
    # Dropout
    norm2 = tf.nn.dropout(norm2, _dropout)

    # 卷积
    conv3 = conv2d(norm2, _weights['wc3'], _biases['bc3'])
    # 下采样
    pool3 = max_pool('pool3', conv3, k=2)
    # 归一化
    norm3 = norm('norm3', pool3, lsize=4)
    # Dropout
    norm3 = tf.nn.dropout(norm3, _dropout)

    # 全连接层，先把特征图转为向量
    dense1 = tf.reshape(norm3, [-1, _weights['wd1'].get_shape().as_list()[0]])
    dense1 = tf.nn.relu(tf.matmul(dense1, _weights['wd1']) + _biases['bd1'], name='fc1')
    # 全连接层
    dense2 = tf.nn.relu(tf.matmul(dense1, _weights['wd2']) + _biases['bd2'], name='fc2') # Relu activation

    # 网络输出层
    out = tf.matmul(dense2, _weights['out']) + _biases['out']
    return out

"""
        User defined parameter
"""

#global setting
image_size =(224,224)

# image anti-clockwise rotation angle in preprocess phase
theta = 0

#flip
flip_mode = None  # None : None, 0 : vertical flip , positive : horizontal flip, negative: horizontal and vertical

#add noise model in preprocess phase
noise_model = 1  #0 : NONE, 1: Gaussian 2: salt and pepper noise
noise_mean = 0.0
noise_sigma  = 20.0
salt_percent = 0.0

#board

#image scan directory setting
#training_set_dir = r'C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\datasets\training_data_set'
#test_set_dir = r'C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\datasets\test_data_set'
training_set_dir = r'C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\datasets\overfit'
test_set_dir = r'C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\datasets\overfit'
current_file_id = 0
model_path = r"C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\models"
model_name = r"C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\models\model.ckpt"
logs_path = r'C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\board'
img_path = r"C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\output"
log_file  = r"C:\Nvidia\my_library\visualSearch\TNR\github\denoising_dl\output.log"
"""
training_set_dir = r'/home/pyp/paper/denosing/denoising_dl/training_data'
test_set_dir = r'/home/pyp/paper/denosing/denoising_dl/test_data'
img_path = r'/home/pyp/paper/denosing/denoising_dl/output'
model_path = r'/home/pyp/paper/denosing/denoising_dl/models'
model_name = r'/home/pyp/paper/denosing/denoising_dl/models/model.ckpt'
logs_path = r'/home/pyp/paper/denosing/denoising_dl/board'
log_file  = r'/home/pyp/paper/denosing/denoising_dl/output.log'
"""
#random training sets
seed = 0    #fixed order with fixed seed

"""
    Internal variable , no setting by user
"""
#internal global variable, no setting by user
patch_current_x = 0
patch_current_y = 0
current_image = None
current_image_true = None

"""
 ---------------------------------------------------------
                  MLP control parameters
"""
tf.reset_default_graph()
learning_period = 30
learning_ratio = 1.0
training_epochs = 100
batch_size = 128
num_examples = 6000
display_step = 1
threshold_adjust = 0.90
early_termination_threshold = 1/100000
mode = 1 #mean,stddev, 1: min,max
#continuous traing or training from scatch
#training_mode = "test_only" # continuous,test_only
training_mode = "continuous"
reset_learning_rate_enable = 0
reset_learning_rate = 0.00001  #when rerun the epoch, if need to reset the learning rate
learning_rate_threshold = 0.001 #not below the value for learning rate
channel = 1
save_step = 20
prev_cost = 0
network = "ALEXNET"

#patch parameters

if network == "MLP" :
    patch_size = (14,14)
    patch_stride = 14
    n_input = patch_size[0]*patch_size[1] # MNIST data input (img shape: 28*28)
    n_output = patch_size[0]*patch_size[1] # denoised patch size (img shape: 28*28)
    # Network Parameters
    n_hidden_1 = 256 # 1st layer number of features
    n_hidden_2 = 256 # 2nd layer number of features
    n_hidden_3 = 256 # 3nd layer number of features
    dropout = 0.000001 # Dropout, probability to keep units
    active_mode = "Relu" #Sigmoid,Tanh
    #Relu use He weight initialization
    if active_mode == "Relu":
        weights = {
            'h1': tf.Variable(np.random.randn(n_input,n_hidden_1)/np.sqrt(n_input/2),dtype="float32"),
            'h2': tf.Variable(np.random.randn(n_hidden_1,n_hidden_2)/np.sqrt(n_hidden_1/2),dtype="float32"),
            'h3': tf.Variable(np.random.randn(n_hidden_2,n_hidden_3)/np.sqrt(n_hidden_2/2),dtype="float32"),
            'out': tf.Variable(np.random.randn(n_hidden_3,n_output)/np.sqrt(n_hidden_3/2),dtype="float32")
            }
        biases = {
            'b1': tf.Variable(tf.random_normal([n_hidden_1])),
            'b2': tf.Variable(tf.random_normal([n_hidden_2])),
            'b3': tf.Variable(tf.random_normal([n_hidden_3])),
            'out': tf.Variable(tf.random_normal([n_output]))
        }
    elif active_mode == "Sigmoid": #sqrt(6/(fan_in+fan_out))
        weights = {
            'h1': tf.Variable(tf.random_uniform([n_input, n_hidden_1],minval=-np.sqrt(6/(n_input+n_hidden_1)),maxval=np.sqrt(6/(n_input+n_hidden_1)),dtype=tf.float32)),
            'h2': tf.Variable(tf.random_uniform([n_hidden_1, n_hidden_2],minval=-np.sqrt(6/(n_hidden_2+n_hidden_1)),maxval=np.sqrt(6/(n_hidden_2+n_hidden_1)),dtype=tf.float32)),
            'h3': tf.Variable(tf.random_uniform([n_hidden_2, n_hidden_3],minval=-np.sqrt(6/(n_hidden_2+n_hidden_3)),maxval=np.sqrt(6/(n_hidden_2+n_hidden_3)),dtype=tf.float32)),
            'out': tf.Variable(tf.random_uniform([n_hidden_3, n_output],minval=-np.sqrt(6/(n_hidden_3+n_output)),maxval=np.sqrt(6/(n_hidden_3+n_output)),dtype=tf.float32))
            }
        biases = {
            'b1': tf.Variable(tf.random_normal([n_hidden_1])),
            'b2': tf.Variable(tf.random_normal([n_hidden_2])),
            'b3': tf.Variable(tf.random_normal([n_hidden_3])),
            'out': tf.Variable(tf.random_normal([n_output]))
        }
    else: #tanh set 4*sqrt(6/(fan_in+fan_out))
        weights = {
            'h1': tf.Variable(tf.random_uniform([n_input, n_hidden_1],minval=-4.0*np.sqrt(6/(n_input+n_hidden_1)),maxval=4.0*np.sqrt(6/(n_input+n_hidden_1)),dtype=tf.float32)),
            'h2': tf.Variable(tf.random_uniform([n_hidden_1, n_hidden_2],minval=-4.0*np.sqrt(6/(n_hidden_2+n_hidden_1)),maxval=4.0*np.sqrt(6/(n_hidden_2+n_hidden_1)),dtype=tf.float32)),
            'h3': tf.Variable(tf.random_uniform([n_hidden_2, n_hidden_3],minval=-4.0*np.sqrt(6/(n_hidden_2+n_hidden_3)),maxval=4.0*np.sqrt(6/(n_hidden_2+n_hidden_3)),dtype=tf.float32)),
            'out': tf.Variable(tf.random_uniform([n_hidden_3, n_output],minval=-4.0*np.sqrt(6/(n_hidden_3+n_output)),maxval=4.0*np.sqrt(6/(n_hidden_3+n_output)),dtype=tf.float32))
        }
        biases = {
            'b1': tf.Variable(tf.random_normal([n_hidden_1])),
            'b2': tf.Variable(tf.random_normal([n_hidden_2])),
            'b3': tf.Variable(tf.random_normal([n_hidden_3])),
            'out': tf.Variable(tf.random_normal([n_output]))
        }
elif network == "CNN" or network == "CNNBATCH": #with good weight initialization and batch normalization layer
    patch_size = (28,28)
    patch_stride = 14
    n_input = patch_size[0]*patch_size[1] # MNIST data input (img shape: 28*28)
    n_output = patch_size[0]*patch_size[1] # denoised patch size (img shape: 28*28)
    dropout = 0.5 # Dropout, probability to keep units
    W_init = tf.contrib.layers.xavier_initializer_conv2d()
    weights = {
        # 5x5 conv, 1 input, 32 outputs
        'h1': tf.get_variable('h1', [5, 5, 1, 32], initializer=W_init),
        # 5x5 conv, 32 inputs, 64 outputs
        'h2': tf.get_variable('h2', [5, 5, 32, 64], initializer=W_init),
        # fully connected, 7*7*64 inputs, 1024 outputs
        'h3': tf.get_variable('h3', [7*7*64, 1024], initializer=W_init),
        # 1024 inputs, 10 outputs (class prediction)
        'out': tf.get_variable('out', [1024, n_output], initializer=W_init)
     }

    biases = {
        'b1': tf.Variable(tf.random_normal([32])),
        'b2': tf.Variable(tf.random_normal([64])),
        'b3': tf.Variable(tf.random_normal([1024])),
        'out': tf.Variable(tf.random_normal([n_output]))
    }

elif network == "CNNBATCH1": #no good weight optimization with batch normalization layer
    patch_size = (28,28)
    patch_stride = 14
    n_input = patch_size[0]*patch_size[1] # MNIST data input (img shape: 28*28)
    n_output = patch_size[0]*patch_size[1] # denoised patch size (img shape: 28*28)

    weights = {
        # 5x5 conv, 1 input, 32 outputs
        'h1': tf.Variable(tf.random_normal([5, 5, 1, 32])),
        # 5x5 conv, 32 inputs, 64 outputs
        'h2': tf.Variable(tf.random_normal([5, 5, 32, 64])),
        # fully connected, 7*7*64 inputs, 1024 outputs
        'h3': tf.Variable(tf.random_normal([7*7*64, 1024])),
        # 1024 inputs, 10 outputs (class prediction)
        'out': tf.Variable(tf.random_normal([1024, n_output]))
     }

    biases = {
        'b1': tf.Variable(tf.random_normal([32])),
        'b2': tf.Variable(tf.random_normal([64])),
        'b3': tf.Variable(tf.random_normal([1024])),
        'out': tf.Variable(tf.random_normal([n_output]))
    }

    batchN_scale = {
        'b1': tf.Variable(tf.ones([32])),
        'b2': tf.Variable(tf.ones([64])),
        'b3': tf.Variable(tf.ones([1024]))
     }

    batchN_beta = {
        'b1': tf.Variable(tf.ones([32])),
        'b2': tf.Variable(tf.ones([64])),
        'b3': tf.Variable(tf.ones([1024]))
     }

elif network == "ALEXNET":
    patch_size = (28,28)
    patch_stride = 14
    n_input = patch_size[0]*patch_size[1] # MNIST data input (img shape: 28*28)
    n_output = patch_size[0]*patch_size[1] # denoised patch size (img shape: 28*28)
    dropout = 0.5 # Dropout, probability to keep units
    W_init = tf.contrib.layers.xavier_initializer_conv2d()

    weights = {
        # 5x5 conv, 1 input, 32 outputs
        'wc1': tf.get_variable('wc1', [3, 3, 1, 64], initializer=W_init),
        # 5x5 conv, 32 inputs, 64 outputs
        'wc2': tf.get_variable('wc2', [3, 3, 64, 128], initializer=W_init),
        # fully connected, 7*7*64 inputs, 1024 outputs
        'wc3': tf.get_variable('wc3', [3, 3, 128, 256], initializer=W_init),
        'wd1': tf.get_variable('wd1', [4*4*256, 1024], initializer=W_init),
        'wd2': tf.get_variable('wd2', [1024, 1024], initializer=W_init),
        # 1024 inputs, 10 outputs (class prediction)
        'out': tf.get_variable('out', [1024, n_output], initializer=W_init)
     }

    """
    weights = {
     'wc1': tf.Variable(tf.random_normal([3, 3, 1, 64])),
     'wc2': tf.Variable(tf.random_normal([3, 3, 64, 128])),
     'wc3': tf.Variable(tf.random_normal([3, 3, 128, 256])),
     'wd1': tf.Variable(tf.random_normal([4*4*256, 1024])),
     'wd2': tf.Variable(tf.random_normal([1024, 1024])),
     'out': tf.Variable(tf.random_normal([1024, n_output]))
    }
    """
    biases = {
     'bc1': tf.Variable(tf.random_normal([64])),
     'bc2': tf.Variable(tf.random_normal([128])),
     'bc3': tf.Variable(tf.random_normal([256])),
     'bd1': tf.Variable(tf.random_normal([1024])),
     'bd2': tf.Variable(tf.random_normal([1024])),
     'out': tf.Variable(tf.random_normal([n_output]))
    }

#Vriable defines which can be save/restore by saver
learning_rate = tf.Variable(0.001,dtype="float",name="learning_rate")

seed = time.time()
print("random seed is :", seed)
tf.set_random_seed(seed)
random.seed(seed)

#set weight initialization range for different activate
weight_init_min = 0.0
weight_init_max = 2.0

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_output])
keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)
phase = tf.placeholder(tf.bool, name='phase')
#lrate = tf.placeholder("float",[1])

#test program
result = scan_image_directories(training_set_dir)
result = random_image_list(result,seed)

result_test = scan_image_directories(test_set_dir)
result_test = random_image_list(result_test,seed)

# Construct model
if network == "MLP":
    pred = multilayer_perceptron(x, weights, biases,active_mode)
elif network == "CNN":
    pred = conv_net(x, weights, biases, keep_prob)
elif network == "CNNBATCH" or network == "CNNBATCH1":
    pred = conv_net_batch(x, weights, biases,phase)
elif network == "ALEXNET":
    pred = alex_net(x, weights, biases, keep_prob)

# Define loss and optimizer
cost = tf.nn.l2_loss(pred-y)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
    #optimizer = tf.train.GradientDescentOptimizer(0.01).minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
#init = tf.initialize_all_variables()
saver = tf.train.Saver()

#tensorboard
#tf.summary.histogram("weight1",weights['h1'])
#tf.summary.histogram("weight2",weights['h2'])
#tf.summary.histogram("weight3",weights['h3'])
#tf.summary.histogram("weightout",weights['out'])
#tf.summary.histogram("bias1",biases['b1'])
#tf.summary.histogram("bias2",biases['b2'])
#tf.summary.histogram("bias3",biases['b3'])
#tf.summary.histogram("biasout",biases['out'])
tf.summary.scalar("loss",cost)
merged_summary_op = tf.summary.merge_all()

f_log = open(log_file,"w")

# Launch the graph for training phase
if training_mode != "test_only":
    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
        sess.run(init)
        summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())
        if training_mode == "continuous" :
            ckpt = tf.train.get_checkpoint_state(model_path)
            if ckpt and ckpt.model_checkpoint_path:
                print("continuous mode is enabled, the cp is :",ckpt.model_checkpoint_path)
                saver.restore(sess,ckpt.model_checkpoint_path)
                if reset_learning_rate_enable == 1:
                    print("Reset learning rate after restore when the learning rate is too small for converge!")
                    sess.run(tf.assign(learning_rate,reset_learning_rate))

        # Training cycle
        print("Start training!")
        for epoch in range(training_epochs):
            avg_cost = 0.
            current_file_id = 0 # reset patch read index
            total_batch = int(num_examples/batch_size)
            #data augment for theta rotation
            if epoch % learning_period == learning_period - 1 :
              #  theta = np.random.uniform(-180,180)
                print("epoch:",epoch+1," the theta is:",theta)
            # Loop over all batches
            for i in range(total_batch):
                batch_x, batch_y = next_batch(result,batch_size)

                if batch_y is None:
                    break
                # Run optimization op (backprop) and cost op (to get loss value)
                if network == "MLP" or network == "CNNBATCH" or network == "CNNBATCH1" :
                    _, c,summary = sess.run([optimizer,cost,merged_summary_op], feed_dict={x: batch_x,
                                                          y: batch_y,phase:True,keep_prob:dropout
                                                          })
                elif network == "CNN"  or network == "ALEXNET":
                    _, c,summary = sess.run([optimizer,cost,merged_summary_op], feed_dict={x: batch_x,
                                                          y: batch_y,keep_prob:dropout
                                                          })

               #image_recovery(image_size[0],image_size[1],patch_size[0],patch_size[1],patch_stride,batch_y)
               # Compute average loss
                summary_writer.add_summary(summary,epoch*total_batch+i)
                avg_cost += c/num_examples
                # Display logs per epoch step
            if epoch % learning_period == 0:
                print("current learning rate is :",sess.run(learning_rate))
                prev_cost = avg_cost
            """
            if avg_cost > prev_cost*threshold_adjust  and epoch % learning_period == learning_period - 1:
                 if abs(prev_cost - avg_cost) < early_termination_threshold*prev_cost:
                     print("Early termination!",prev_cost,avg_cost)
                     #break
                 if sess.run(learning_rate) > learning_rate_threshold :
                     sess.run(tf.assign(learning_rate,learning_rate *learning_ratio))
                     print("Adjust learning rate to ",sess.run(learning_rate))
            """
            if epoch % display_step == 0:
                print("Epoch:", '%04d' % (epoch+1), "cost=", \
                    "{:.9f}".format(avg_cost),",weight_out:",sess.run(tf.nn.moments(tf.reshape(weights['out'],[-1]),axes=[0])),",bias out:",sess.run(tf.nn.moments(tf.reshape(biases['out'],[-1]),axes=[0])))

            if (epoch + 1)% save_step == 0 :
                print("Save model in epoch ",epoch+1, " learning rate is:",sess.run(learning_rate))
                saver.save(sess,model_name,global_step = epoch + 1)

        #save the last step
        print("Save model in epoch ",epoch+1, " learning rate is:",sess.run(learning_rate))
        saver.save(sess,model_name,global_step = epoch+1)
        print("Training weights out :",sess.run(weights['out']), "bias out:",sess.run(biases['out']))
        print("Optimization Finished!")

#Test phase
with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
    print("Start test phase for one image!")
    ckpt = tf.train.get_checkpoint_state(model_path)
    if ckpt and ckpt.model_checkpoint_path:
        print("Test mode is enabled, load the cp is :",ckpt.model_checkpoint_path)
        saver.restore(sess,ckpt.model_checkpoint_path)
        print("Restore model and learning rate is:",sess.run(learning_rate))

    #reset theta to zero in test phase
    theta = 0
    for i in range(len(result_test)):
        test_image = result_test[i]
        print("test image is:",test_image)
        f_log.write("test image is %s\n" %(test_image))
        patch_current_y = 0
        patch_current_x = 0
        batch_x,batch_y,p0,p1,patch_num= get_patches_one_image(test_image)
        if network == "MLP" or network == "CNNBATCH":
            patch_recover,cost_test = sess.run([pred,cost],{x:batch_x,y:batch_y,phase:False,keep_prob:1.})
        elif network == "CNN" or network=="ALEXNET":
            patch_recover,cost_test = sess.run([pred,cost],{x:batch_x,y:batch_y,keep_prob: 1.})
        #print(patch_recover)
        print("Test phase: cost = ", cost_test/patch_num)
        f_log.write("Test phase: cost = %f" %(cost_test/patch_num))
        #print("Test weights out :",sess.run(weights['out']), "bias out:",sess.run(biases['out']))
        #print(sess.run(tf.reduce_max(patch_recover)),sess.run(tf.reduce_max(weights['h1'])))
        frame_denoise = image_recovery(image_size[0],image_size[1],patch_size[0],patch_size[1],patch_stride,patch_recover)
        frame_denoise = image_renorm(frame_denoise,p0,p1,mode)
        frame_noise = image_recovery(image_size[0],image_size[1],patch_size[0],patch_size[1],patch_stride,batch_x)
        frame_noise = image_renorm(frame_noise,p0,p1,mode)
        golden_image = get_golden_image_show(test_image)

        #Call BM3D as the denoise reference

        cv2.imwrite(img_path+str(i)+".jpg",frame_denoise)
        cv2.imwrite(img_path+str(i)+"_golden.jpg",golden_image)
        cv2.imwrite(img_path+str(i)+"_noise.jpg",frame_noise)
        #print("the real frame cost:",sess.run(tf.nn.l2_loss(frame-golden_image)))
        plt.subplot(1,2,1)
        plt.imshow(frame_denoise,cmap='gray')
        plt.subplot(1,2,2)
        plt.imshow(golden_image,cmap='gray')
        plt.show()

        #print image quality
        noise_psnr = evaluate_image("PSNR",golden_image.astype(np.uint8),frame_noise.astype(np.uint8))
        noise_ssim = evaluate_image("SSIM",golden_image.astype(np.uint8),frame_noise.astype(np.uint8))
        denoise_psnr = evaluate_image("PSNR",golden_image.astype(np.uint8),frame_denoise.astype(np.uint8))
        denoise_ssim = evaluate_image("SSIM",golden_image.astype(np.uint8),frame_denoise.astype(np.uint8))
        print("After denoising, the psnr is from %f to %f, ssim from %f to %f :" %(noise_psnr,denoise_psnr,noise_ssim,denoise_ssim))

    f_log.close()


t=int(input())
for _ in range(t):
    l=list(map(int,input().split()))
    cnt=0
    for i in range(min(l)+1):
        l.sort(reverse=True)
        if l[2]==0:
            print(cnt+l[1])
        else:
            l[0]-=1
            l[2]-=1
            cnt+=1


from typing import List
import pdb

solution_json = {
    "date": "2021/4/22",
    "coding": 7,
    "bug": "Time Limit Exceeded"
}

class Solution:
    def strStr(self, haystack: str, needle: str) -> int:
    	d = False

    	if needle == "":
    		return 0

    	n = len(haystack)
    	m = len(needle)
    	out = -1
    	for i in range(n):
    		c = haystack[i]

    		matched = True
    		for j in range(m):
    			if i + j >= n:
    				matched = False
    				break
    			if haystack[i + j] != needle[j]:
    				matched = False
    				break

    		if d:
    			print(i, c, matched)

    		if matched:
    			out = i
    			break

    	#pdb.set_trace()
    	return out


from calvin.actor.actor import Actor, manage, condition, calvinlib
from calvin.runtime.north.calvin_token import EOSToken, ExceptionToken

class FromString(Actor):
    """
    Transform JSON-formatted string to value

    Invalid input will produce an Exception token as output unless another value is supplied
    through the optional 'exception_output' argument.
    N.B. Using 'null' for 'exception_output' will produce an ExceptionToken rather than 'null'.

    Inputs:
      string : JSON-formatted string
    Outputs:
      data   : data read from input string
    """

    def exception_handler(self, action, args):
        return (self.default,)

    @manage(['exception_output'])
    def init(self, exception_output=None):
        self.exception_output = exception_output
        self.setup()

    def did_migrate(self):
        self.setup()

    def setup(self):
        self.default = ExceptionToken() if self.exception_output is None else self.exception_output
        self.json = calvinlib.use('json')

    @condition(['string'], ['data'])
    def load(self, string):
        try:
            res = self.json.fromstring(string)
        except:
            res = self.default
        return (res,)

    action_priority = (load,)
    requires = ['json']

    test_set = [
        {
            'inports': {'string': ['1']},
            'outports': {'data': [1]},
        },
        {
            'inports': {'string': ['{"a": 1}']},
            'outports': {'data': [{"a": 1}]},
        },
        {
            'inports': {'string': [EOSToken()]},
            'outports': {'data': ['Exception']},
        },
        {
            'inports': {'string': [None]},
            'outports': {'data': ['Exception']},
        },
        {
            'setup': [lambda self: self.init(exception_output={})],
            'inports': {'string': [None]},
            'outports': {'data': [{}]},
        },
    ]


from ctypes import CDLL, c_char

def get_cert():
    dll = CDLL("c_/lib_anwei.so")
    cert_list = (c_char * 1024)()
    dll.export_cert(cert_list)
    cert = str(bytes.hex(cert_list.raw).encode("utf-8"), "utf-8")
    return cert

a = get_cert()
print(a)


values = [0, 25, 50, 41, 30]
valuex = [0, 0, 25, 50, 41, 30]
order = ["first", "second", "third", "fourth", "fifth"]
sum = 0
summes = [0]
length = 0
for value in values:
    sum += value
    length += 1
    if sum == None:
        sum = 0
    else:
        sum = sum
    index = int(length)
    summes.append(sum)
    average = sum / length
    print(
        f"In the {order[index-1]} try the sum of {summes[index-1]} and {values[index-1]} was {sum} and the average of this {length} items was {average} \n"
    )


import numpy as np
import cv2
import matplotlib.pyplot as plt

#Import necessary functions
from matchPics import matchPics
from helper import loadVid
from opts import get_opts
from planarH import computeH_ransac
from planarH import compositeH
#from helper import plotMatches

opts = get_opts()

#Write script for Q3.1

book = loadVid("../data/book.mov")
shape_book = np.shape(book)

cv_cover = cv2.imread('../data/cv_cover.jpg')
shape_cover = cv_cover.shape

ars = loadVid("../data/ar_source.mov")
ars = ars[:,45:315,:,:] ## Removes the black strips from top and bottom
shape_ars = np.shape(ars)
### the range of 40 to 320 is found by the for loop below
###for i in range(shape_ars[1]):
    ###print(i, ars[1,i,100,1])

    # 0.8 15 1000 2
centerx = shape_ars[2]//2
x = centerx - shape_cover[1]/2

ars = ars[:,:, int(x):int(x+shape_cover[1]),:] ### central region extracted from ars
shape_ars = np.shape(ars)

f = min(shape_ars[0],shape_book[0])
frame_size = (shape_book[2],shape_book[1])

out = cv2.VideoWriter('../result/ar.avi',cv2.VideoWriter_fourcc('M','J','P','G'), 30, frame_size)

for i in range(f):
    print(i)
    ars_frame = cv2.resize(ars[i,:,:,:], (cv_cover.shape[1], cv_cover.shape[0]))
    ars_frame_gray = cv2.cvtColor(ars_frame,cv2.COLOR_BGR2GRAY)
    book_frame = book[i,:,:,:]
    book_frame_gray = cv2.cvtColor(book_frame,cv2.COLOR_BGR2GRAY)
    cv_cover_gray = cv2.cvtColor(cv_cover,cv2.COLOR_BGR2GRAY)

    matches, locs1, locs2 = matchPics(book_frame, cv_cover, opts)
    #plotMatches(cv_desk, cv_cover, matches, locs1, locs2) ### already prints only true matches

    locs1 = locs1[matches[:,0]]
    locs1 = locs1[:,[1,0]]
    locs2 = locs2[matches[:,1]]
    locs2 = locs2[:,[1,0]]
    bestH2to1, inliers = computeH_ransac(locs1, locs2, opts) #tell us how to go from locs2 to locs1

    comp_img = compositeH(bestH2to1, ars_frame, book_frame)

    dst = cv2.cvtColor(comp_img, cv2.COLOR_BGR2RGB)
    #plt.imshow(dst)

    #path = '../images/'+str(i)+'.png'
    #plt.imsave(path, dst)
    out.write(dst[:,:,::-1])

out.release()


import unittest

import pytest

from lidarrmetadata import util

@pytest.mark.parametrize('dictionary,key,default,expected', [
    ({}, 'a', None, None),  # Default value
    ({}, 'a', 1, 1),  # Pass default value
    ({'a': [1, 2]}, 'a', None, 1),  # Normal usage
    ({'a': 1}, 'a', None, 1),  # Not list
    ({'a': [[1, 2], [3, 4]]}, 'a', None, [1, 2]),  # Nested list
])
def test_first_key_item(dictionary, key, default, expected):
    assert expected == util.first_key_item(dictionary, key, default)

@pytest.mark.parametrize('iterable,func,types,expected', [
    ({'a': 1}, lambda _: 2, object, {'a': 2}),
    ({'a': 1}, lambda _: 2, str, {'a': 1}),
    ({'a': 1, 2: 'b'}, lambda s: s.upper(), str, {'a': 1, 2: 'B'}),
    ([1, 2, 3], lambda i: i + 1, int, [2, 3, 4]),
    ((1, 2, 3), lambda i: i + 1, object, (2, 3, 4)),
    (['a', 1, [1, 2]], lambda _: 2, int, ['a', 2, [2, 2]]),
    ([[1, 2]], lambda i: i + 1, int, [[2, 3]]),
    (['a', 1, 1.1], lambda _: 2, [int, float], ['a', 2, 2])
])
def test_map_iterable_values(iterable, func, types, expected):
    result = util.map_iterable_values(iterable, func, types)
    assert expected == result

@pytest.mark.parametrize('string,table,expected', [
    ('abc', {'a': 'q'}, 'qbc'),
    ('abc', {'a': 'qw'}, 'qwbc'),
    ('abc', {'ab': 'QW'}, 'abc')  # Doesn't support 2-character in source at the moment
])
def test_translate_string(string, table, expected):
    result = util.translate_string(string, table)
    assert expected == result

class BidirectionalDictionaryTest(unittest.TestCase):
    def setUp(self):
        self.d = util.BidirectionalDictionary({'a': 1, 'b': 2})

    def test_regular_access(self):
        assert self.d['a'] == 1
        assert self.d['b'] == 2

    def test_regular_set(self):
        assert self.d['a'] == 1
        self.d['a'] = 2
        assert self.d['a'] == 2

    def test_inverse_access(self):
        assert self.d.inverse[1] == 'a'
        assert self.d.inverse[2] == 'b'

    def test_inverse_set(self):
        assert self.d.inverse[1] == 'a'
        self.d['a'] = 2
        assert self.d.inverse[2] == 'a'


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.interpolate import make_interp_spline, BSpline
import numpy as np

encoders = ["None","resnet50", "resnet101", "resnext50_32x4d", "resnext101_32x8d", "densenet121", "densenet201",
                "pre_trained_resnet50", "pre_trained_resnet101", "pre_trained_resnext50_32x4d", "pre_trained_resnext101_32x8d",
                "pre_trained_densenet121", "pre_trained_densenet201"]

""" Plotting training and validation loss for each encoder """
sns.set_style("whitegrid", {'grid.linestyle': '--'})
f, ax = plt.subplots(figsize=(18, 10))
best_loss = float("inf")
for encoder_name in encoders:
    model_name = "Unet_" + encoder_name
    df = pd.read_csv("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/files/train_data/"+model_name+".csv")
    x = df["Epoch"]
    y = df["Train"]
    xnew = np.linspace(x.min(), x.max(), 200)

    spl = make_interp_spline(x, y, k = 3)
    ynew = spl(xnew)

    if best_loss > df["Train"].min():
        best_loss = df["Train"].min()
        best_model = model_name
    fig = sns.lineplot(x = xnew, y = ynew, label = encoder_name)
    #print(df["Train"].min(), best_model)
fig.set(xlabel="Epoch", ylabel="Loss", title="Train Loss per Epoch", xlim=(0, 19), xticks=list(range(0, 20)))
ax.text(2, 1.2,"Best Encoder: "+best_model+"\nwith train loss: "+str(best_loss), fontsize=10)
plt.savefig("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/assets/Train_loss.png", dpi=300)

f, ax = plt.subplots(figsize=(18, 10))
best_loss = float("inf")
for encoder_name in encoders:
    model_name = "Unet_" + encoder_name
    df = pd.read_csv("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/files/train_data/"+model_name+".csv")
    x = df["Epoch"]
    y = df["Valid"]
    xnew = np.linspace(x.min(), x.max(), 200)

    spl = make_interp_spline(x, y, k = 3)
    ynew = spl(xnew)

    if best_loss > df["Valid"].min():
        best_loss = df["Valid"].min()
        best_model = model_name
    fig = sns.lineplot(x = xnew, y = ynew, label = encoder_name)
    #print(df["Valid"].min(), best_model)
    fig = sns.lineplot(x = xnew, y = ynew, label = encoder_name)
fig.set(xlabel="Epoch", ylabel="Loss", title="Evaluation Loss per Epoch", xlim=(0, 19), xticks=list(range(0, 20)))
ax.text(7, 1.7,"Best Encoder: "+best_model+"\nwith valid loss: "+str(best_loss), fontsize=10)
plt.savefig("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/assets/Evaluation_loss.png", dpi=300)

""" Evaluation metrics for each encoder """

"""Reading csv file into dataframe"""
fig, ax = plt.subplots(figsize=(18, 10))
results = pd.read_csv("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/results/metrics.csv")
ax = sns.barplot(x="Encoder", y="Value", hue= "score category", data=results)
ax.set(xlabel="Encoder", ylabel="Value", title="Evaluation Metrics")
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig("/home/surya/projects/Breast-Cancer-Segmentation-using-Deep-Learning/assets/metrics.png", dpi=300)
plt.show()


import textrazor
from googletrans import Translator
import resources.creds as credentials

class Topic():

    def __init__(self, text):
        self._text = text
        self._topics_from_txt = set()
        self._topic = self._topics()
        self._topic_in_dic(self._topic)
        self._translate_topics()

    @property
    def getTopics(self):
        return self._topics_from_txt

    def _topics(self):
        textrazor.api_key = credentials.api_key
        client = textrazor.TextRazor(extractors=["entities", "topics"])
        response = client.analyze(self._text)
        return response

    def _topic_in_dic(self, response_topics):

        for entity in response_topics.topics():
            if(0.5<=entity.score <=1):
                self._topics_from_txt.add((entity.label.lower(), entity.score))
            else: break

    def _translate_topics(self):

        topics = self._topics_from_txt
        self._topics_from_txt = set()
        translator = Translator()

        for x in topics:
            translation = translator.translate(x[0], dest='es')
            self._topics_from_txt.add((translation.text.lower(), x[1]))


# расчет начального вектора pi
def get_vector_pi(matrix, q):

    pi = []
    l = len(matrix[0])

    for i in range(0, l):
        if i == q[0]:
            pi.append(1)
        else:
            pi.append(0)

    return pi


# 90. 子集 II
# 给定一个可能包含重复元素的整数数组 nums，返回该数组所有可能的子集（幂集）。
# 说明：解集不能包含重复的子集。

class Solution:
    def subsetsWithDup(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        size = len(nums)
        if size == 0:
            return []
        nums.sort()
        res = []
        self.__dfs(nums, 0, size, [], res)
        return res

    def __dfs(self, nums, start, size, path, res):
        res.append(path[:])
        for i in range(start, size):
            if i > start and nums[i - 1] == nums[i]:
                continue
            path.append(nums[i])
            self.__dfs(nums, i + 1, size, path, res)
            path.pop()

if __name__ == '__main__':
    nums = [1, 2, 2]
    solution = Solution()
    result = solution.subsetsWithDup(nums)
    print(result)


import logging
import torch

from .autoencoder import TCNAutoencoder
from .supervised import SupervisedTCN

import json
from datetime import datetime, date

class ModelManager:
    def __init__(self, args, training=True, **kwargs):
        self.is_training = training
        self.load_network(args, **kwargs)

    def load_network(self, args, **kwargs):
        if args.model_pos or args.model_vel:
            print('Load network')
            if self.is_training:
                self.network, self.optim = self._load_pretrained_model(args.model_pos or args.model_vel)
            else:
                self.network = {}
                vel_net = self._load_pretrained_model(args.model_vel)
                if vel_net:
                    self.network['vel'] = vel_net.eval()
                pos_net = self._load_pretrained_model(args.model_pos)
                if pos_net:
                    self.network['pos'] = pos_net.eval()
        #erstelle Netzwerk
        else:
            self.network = TCNAutoencoder(args)

            params_basis = [*self.network.encoder.parameters(), *self.network.decoder.parameters()]
            params_adversarial = [*self.network.adversary_decoder.parameters()]

            self.optim_basis = torch.optim.Adam(params_basis, lr=args.learning_rate)
            self.optim_adversarial = torch.optim.Adam(params_adversarial, lr=args.learning_rate)

            self._log(self.network)

    def _load_pretrained_model(self, model_name):
        if not model_name:
            return None

        logging.info('Loading saved model {}...'.format(model_name))
        model = torch.load('../models/' + model_name)
        try:
            network = model['network']
        except KeyError:
            network = model['model']
        self._log(network)

        network.load_state_dict(model['model_state_dict'])

        if self.is_training:
            optim = torch.optim.Adam(network.parameters())
            optim.load_state_dict(model['optimizer_state_dict'])
            return network, optim

        return network

    def _log(self, network):
        #TODO prints network out: logging.info('\n ' + str(network))
        logging.info('# of Parameters: ' +
                     str(sum(p.numel() for p in network.parameters()
                             if p.requires_grad)))

    def save(self, e, run_identifier, losses, losses_100, name_run, args, is_adv=False):
        model_filename = '../models/' + args.signal_type + '-e' + str(e) + '-hz' + str(args.hz)
        torch.save(
            {
                'epoch': e,
                'network': self.network,
                'model_state_dict': self.network.state_dict(),
                'optimizer_state_dict': self.optim_basis.state_dict(),
                'optimizer_state_dict_adv': self.optim_adversarial.state_dict() if is_adv else [],
                'losses': losses
            }, model_filename)
        logging.info('Model saved to {}'.format(model_filename))

        #save params in extra file
        params = {
            "run_identifier": run_identifier,
            "signal_type"   : args.signal_type,
            "lr"            : args.learning_rate,
            "hz"            : args.hz,
            "viewing time"  : args.viewing_time,
            "bs"            : args.batch_size,
            "slice-time-windows": args.slice_time_windows,
            "last loss"     : list(losses)[-1],
            "current day"   : date.today().strftime("%d.%m.%Y"),
            "current time"  : datetime.now().strftime("%H:%M:%S"),
            "epoch losses train"  : list(losses['train']),
            "epoch losses val"  : list(losses['val']),
            "global losses 100 train" : list(losses_100['train']),
            "global losses 100 val": list(losses_100['val']),
            "CEL epoch losses train"  : list(losses['CELtrain']) if is_adv else [],
            "CEL epoch losses val"  : list(losses['CELval']) if is_adv else [],
            "CEL global losses 100 train" : list(losses_100['CELtrain']) if is_adv else [],
            "CEL global losses 100 val": list(losses_100['CELval'] if is_adv else [])
        }

        jsonFile = json.dumps(params)

        with open(model_filename + '.json', 'w') as jasonfile: # function opens a file, and returns it as a file object., Write - Opens a file for writing, creates the file if it does not exist
            jasonfile.write(jsonFile)


from jax_privacy.experiments.image_data import augmult
import numpy as np
import tensorflow as tf

def decode_large_image(
    image: tf.Tensor,
    *,
    image_size: tuple[int, int],
    augmult_config: augmult.AugmultConfig | None,
) -> tf.Tensor:
  """Decodes the image and returns it with float32 values within [0, 1]."""
  if image.dtype == tf.dtypes.string:
    image = _decode_and_center_crop(
        image_bytes=image,
    )
  # Convert to float32 and rescale to [0, 1] after the decoding.
  image = tf.image.convert_image_dtype(image, np.float32)

  if (augmult_config is not None
      and augmult_config.random_crop
      and augmult_config.augmult):
    # Increase the target size to later take random crops within the image,
    # e.g. 268x268 for 224x224 crops.
    image_size = [int(x * 1.2) for x in image_size]
  image = tf.image.resize(image, image_size, tf.image.ResizeMethod.BICUBIC)

  # NOTE: Bicubic resizes without clamping overshoots. This means values
  # returned will be outside the range [0.0, 1.0].
  return tf.clip_by_value(image, 0.0, 1.0)

def _decode_and_center_crop(
    *,
    image_bytes: tf.Tensor,
) -> tf.Tensor:
  """Decodes a JPEG and takes a square center crop to make the image square."""
  jpeg_shape = tf.image.extract_jpeg_shape(image_bytes)
  image_height = jpeg_shape[0]
  image_width = jpeg_shape[1]
  min_size = tf.minimum(image_height, image_width)

  offset_height = (image_height - min_size) // 2
  offset_width = (image_width - min_size) // 2
  crop_window = tf.stack([offset_height, offset_width, min_size, min_size])
  image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3)
  return image  # dtype uint8


from media.xml.namespaces import Namespaces
from media.data.nouns import PersonalName, noun_dispatcher
from media.data.media.contents.genericv.crew.cast import Cast

class Crew():
    '''
    Main container element for all crew objects.
    '''
    def __init__(self, in_element):
        self.directors = []
        self.editors = []
        self.writers = []
        self.cinemap = []
        self.music = None
        self.cast = None
        if in_element is not None:
            self._process(in_element)

    def _process(self, in_element):
        for child in in_element:
            tagname = Namespaces.ns_strip(child.tag)
            if tagname == 'directors':
                self.directors = Directors.load(child)
            elif tagname == 'editors':
                self.editors = SimpleCrew.load(child, 'editor')
            elif tagname == 'cinemaphotographers':
                self.cinemap = SimpleCrew.load(child, 'cinemaphotographer')
            elif tagname == 'writers':
                self.writers = SimpleCrew.load(child, 'writer')
            elif tagname == 'music':
                self.music = Music(child)
            elif tagname == 'cast':
                self.cast = Cast(child)

class Directors():
    '''
    Simple class method to parse director elements.
    '''
    @classmethod
    def load(cls, in_element):
        ''' Load an array of names for a given element in the XML'''
        out = []
        if in_element is not None:
            for child in in_element:
                tagname = Namespaces.ns_strip(child.tag)
                if tagname == 'director':
                    out.append(PersonalName(child))
        return out

class Writers():
    '''
    Class method to parser writer elements, which
    are a little more complex compared to other
    crew elements.
    '''
    @classmethod
    def load(cls, in_element):
        ''' Load an array of writers based on the XML'''
        out = []
        if in_element is not None:
            for child in in_element:
                tagname = Namespaces.ns_strip(child.tag)
                if tagname == 'writer':
                    out.append(PersonalName(child))

class SimpleCrew():
    '''
    Generic crew loader object, which returns
    a list of Name elements.
    '''
    @classmethod
    def load(cls, in_element, desired_tag):
        '''Load an array of names for a given element in the XML'''
        out = []
        if in_element is not None:
            for child in in_element:
                tagname = Namespaces.ns_strip(child.tag)
                if tagname == desired_tag:
                    out.append(PersonalName(child))
        return out

    @classmethod
    def output(cls, title, in_list):
        '''Special output formatter to string multiple names together'''
        out_string = ""
        if len(in_list) == 1:
            out_string = f"{title}: {in_list[0]}"
        else:
            out_string = f"{title}"
            for name in in_list:
                out_string += f"{name}. "
            out_string = out_string[:-2]
        return out_string

class CrewAssignment():
    '''
    An assignment to a crew position.
    '''
    def __init__(self, tagname, in_element):
        self.title = tagname
        self.uncredited = False
        self.object = noun_dispatcher(in_element)
        if 'uncredited' in in_element.attrib:
            self.uncredited = True

    def __str__(self):
        return str(self.object)

class Music():
    '''
    Container object for all music related staff.
    '''
    def __init__(self, in_element):
        self.allowed = ['composer', 'conductor', 'music']
        self.breakdown = {}
        self.composers = []
        self.conductors = []
        self.music = []
        self._process(in_element)

    def _process(self, in_element):
        for child in in_element:
            tagname = Namespaces.ns_strip(child.tag)
            if tagname in self.allowed:
                # nx_element = child[0]
                crew_obj = CrewAssignment(tagname, child)
                if tagname == 'composer':
                    self.composers.append(crew_obj)
                elif tagname == 'music':
                    self.music.append(crew_obj)
                elif tagname == 'conductor':
                    self.conductors.append(crew_obj)


from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults

class DataQualityOperator(BaseOperator):

    @apply_defaults
    def __init__(self,
                 redshift_conn_id="",
                 tables_list=[],
                 sql_query_test="",
                 *args, **kwargs):

        super(DataQualityOperator, self).__init__(*args, **kwargs)

        self.redshift_conn_id = redshift_conn_id
        self.tables_list = tables_list
        self.sql_query_test = sql_query_test

    def execute(self, context):
        self.log.info('DataQualityOperator is started')

        redshift = PostgresHook(self.redshift_conn_id)

        for table in self.tables_list:
            sql_stmt = self.sql_query_test.format(table)
            self.log.info(f"Data quality check for table <{table}>, SQL = <{sql_stmt}>")
            records = redshift.get_records(sql_stmt)

            if len(records) < 1 or len(records[0]) < 1:
                raise ValueError(f"Data quality check failed. Table <{table}> returned no results.")

            num_records = records[0][0]
            if num_records < 1:
                raise ValueError(f"Data quality check failed. Table <{table}> contains 0 rows.")

            self.log.info(f"Data quality check for table <{table}> passed with {num_records} records.")

        self.log.info('DataQualityOperator is finished.')


from bson import ObjectId
from flask import Flask, jsonify, request
import geopy
from pymongo import MongoClient
from bson import json_util
from geopy import distance
from flask_cors import CORS
import numpy as np
import pandas as pd
import pickle
import sys
import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms
from PIL import Image
import io
import sys

app = Flask(__name__)
CORS(app)
client = MongoClient("mongodb://localhost:27017/")
db = client.c2oufH

@app.route('/recProducts/<product_id>')
def home(product_id):
    product = db.products.find_one({'_id': ObjectId(product_id)})
    product_orders = db.orders.find({'orderItems.product': ObjectId(product_id)})
    product_count = {}
    for order in product_orders:
        for item in order['orderItems']:
            if item['product'] == ObjectId(product_id):
                continue
            if str(item['product']) not in product_count:
                product_count[str(item['product'])] = 0
            product_count[str(item['product'])] += item['qty']
    top_products = sorted(product_count.items(), key=lambda x: x[1], reverse=True)[:5]
    recommended_products = []
    for product_id, count in top_products:
        product = db.products.find_one({'_id': ObjectId(product_id)})
        recommended_products.append(product)
    return json_util.dumps({
        'product': product,
        'recommended_products': recommended_products
    })

@app.route('/nearProducts/<userId>')
def get_nearby_products(userId):
    user = db.users.find_one({"_id": ObjectId(userId)})
    if not user:
        return json_util.dumps({'error': 'User not found'})

    user_location = (user['coordinate']['lat'], user['coordinate']['lng'])
    nearby_products = []

    products = db.products.find()
    for product in products:
        product_user = db.users.find_one({"_id": ObjectId(product['user'])})
        if not product_user:
            return json_util.dumps({'error': 'Product user not found'})

        product_location = (product_user['coordinate']['lat'], product_user['coordinate']['lng'])
        dist = distance.distance(user_location, product_location).km
        if dist <= 10:
            product['user'] = product_user
            nearby_products.append(product)

    return json_util.dumps({'nearby_products': nearby_products})

@app.route('/users/nearbyUsers/<userId>')
def get_nearby_users(userId):
    user = db.users.find_one({"_id": ObjectId(userId)})
    if not user:
        return json_util.dumps({'error': 'User not found'})

    user_location = (user['adresse']['coordinates'][0], user['adresse']['coordinates'][1])
    nearby_users = []

    users = db.users.find()
    for user in users:
        if user['role'] == 'supplier':
            product_location = (user['adresse']['coordinates'][0], user['adresse']['coordinates'][1])
            dist = distance.distance(user_location, product_location).km
            if dist <= 10:
                nearby_users.append(user)

    return json_util.dumps({'nearby_users': nearby_users})

@app.route('/users/otherUsers/<userId>')
def get_other_users(userId):
    user = db.users.find_one({"_id": ObjectId(userId)})
    if not user:
        return jsonify({'error': 'User not found'})

    user_location = (user['coordinate']['lat'], user['coordinate']['lng'])
    nearby_users = []

    users = db.users.find()
    for other_user in users:
        if other_user['_id'] != user['_id'] and other_user['role'] == 'user':
            product_location = (other_user['coordinate']['lat'], other_user['coordinate']['lng'])
            dist = distance.distance(user_location, product_location).km
            if dist <= 10:
                nearby_users.append(other_user)

    # Maintenant, pour chaque utilisateur proche, trouvez ses produits
    for near_user in nearby_users:
        near_user['products'] = list(db.products.find({"user": near_user['_id']}))

    return json_util.dumps({'nearby_users': nearby_users})

@app.route('/rec/<productId>/recommendations')
def recommend_products(productId):
    try:
        orders = db.orders.find({ 'orderItems.product': productId })
        print("Orders:", list(orders))
        products = {}

        for order in orders:
            for item in order['orderItems']:
                if item['product'] != productId:
                    if item['product'] in products:
                        products[item['product']] += item['qty']
                    else:
                        products[item['product']] = item['qty']

        sorted_products = sorted(products.items(), key=lambda x: x[1], reverse=True)[:5]

        recommended_products = []

        for product in sorted_products:
            recommended_product = db.products.find_one({ '_id': product[0] })
            recommended_products.append(recommended_product)

        return jsonify({'recommended_products': recommended_products})
    except Exception as e:
        print(e)
        return jsonify({'message': 'Une erreur est survenue'}), 500

@app.route('/products/<productId>')
def get_product(productId):
    try:
        product = db.products.find_one({ '_id': ObjectId(productId) })
        if product:
            return json_util.dumps(product)
        else:
            return jsonify({'message': 'Le produit n\'existe pas'}), 404
    except Exception as e:
        print(e)
        return jsonify({'message': 'Une erreur est survenue'}), 500

if __name__ == '__main__':
    app.run(debug=True, port=1000)

# python app.py


import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

sns.set(style="whitegrid")

from sklearn.preprocessing import LabelEncoder

from sklearn.model_selection import train_test_split

from nltk.corpus import stopwords

from wordcloud import WordCloud

import collections

import spacy

nlp = spacy.load('en_core_web_sm')

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_selection import SelectPercentile , chi2

from sklearn.naive_bayes import MultinomialNB
data = pd.read_csv('../input/spooky-author-identification/train/train.csv')

print(f'Data Shape is {data.shape}')

data.head()
def show_details() :

    global data

    for col in data.columns :

        print(f'for feature : {col}')

        print(f'Number of Nulls is   {data[col].isna().sum()}')

        print(f'Number of Unique values is   {len(data[col].unique())}')

        print(f'random Value {data[col][0]}')

        print(f'random Value {data[col][10]}')

        print(f'random Value {data[col][20]}')

        print('--------------------------')

def CountWords(text) :

    all_words = []

    for i in range(text.shape[0]) :

        this_phrase = list(text)[i]

        for word in this_phrase.split() :

            all_words.append(word)

    print(f'Total words are {len(all_words)} words')

    print('')

    print(f'Total unique words are {len(set(all_words))} words')

def CommonWords(text ,show = True , kk=10) :

    all_words = []

    for i in range(text.shape[0]) :

        this_phrase = list(text)[i]

        for word in this_phrase.split() :

            all_words.append(word)

    common_words = collections.Counter(all_words).most_common()

    k=0

    word_list =[]

    for word, i in common_words :

        if not word.lower() in  nlp.Defaults.stop_words :

            if show :

                print(f'The word is   {word}   repeated   {i}  times')

            word_list.append(word)

            k+=1

        if k==kk :

            break

    return word_list

def SelectedData(feature , value , operation, selected_feature ):

    global data

    if operation==0 :

        result = data[data[feature]==value][selected_feature]

    elif operation==1 :

        result = data[data[feature] > value][selected_feature]

    elif operation==2 :

        result = data[data[feature]< value][selected_feature]

    return result

def LowerCase(feature , newfeature) :

    global data

    def ApplyLower(text) :

        return text.lower()

    data[newfeature] = data[feature].apply(ApplyLower)

def Drop(feature) :

    global data

    data.drop([feature],axis=1, inplace=True)

    data.head()

def Unique(feature) :

    global data

    print(f'Number of unique vaure are {len(list(data[feature].unique()))} which are : \n {list(data[feature].unique())}')

def Encoder(feature , new_feature, drop = False) :

    global data

    enc  = LabelEncoder()

    enc.fit(data[feature])

    data[new_feature] = enc.transform(data[feature])

    if drop == True :

        data.drop([feature],axis=1, inplace=True)

def MakeCloud(text , title = 'Word Clouds' , w = 15 , h = 15):

    plt.figure(figsize=(w,h))

    plt.imshow(WordCloud(background_color="white",stopwords=set(stopwords.words('english')))

               .generate(" ".join([i for i in text.str.lower()])))

    plt.axis("off")

    plt.title(title)

    plt.show()

def BPlot(feature_1,feature_2) :

    global data

    sns.barplot(x=feature_1, y=feature_2 , data=data)

def CPlot(feature) :

    global data

    sns.countplot(x=feature, data=data,facecolor=(0, 0, 0, 0),linewidth=5,edgecolor=sns.color_palette("dark", 3))

Drop('id')
LowerCase('text' , 'lower text')

Drop('text')
data.head(20)
show_details()
Unique('author')
Encoder('author' , 'author code')

data.head()
CountWords(data['lower text'])
BPlot(data['author'].value_counts().index , data['author'].value_counts().values )
AllCommon = CommonWords(data['lower text'])
MakeCloud(data['lower text'] , 'All Words')
ECommon = CommonWords(SelectedData('author','EAP',0,'lower text'))
MakeCloud(SelectedData('author','EAP',0,'lower text') , 'EAP Words')
HCommon = CommonWords(SelectedData('author','HPL',0,'lower text'))
MakeCloud(SelectedData('author','HPL',0,'lower text') , 'HPL Words')
MCommon = CommonWords(SelectedData('author','MWS',0,'lower text'))
MakeCloud(SelectedData('author','MWS',0,'lower text') , 'MWS Words')
data['number of words'] = data['lower text'].apply(lambda x : len(x.split()))

print('mean words for EAP is  ' , SelectedData('author', 'EAP' , 0 , 'number of words').mean())

print('mean words for HPL is  ' , SelectedData('author', 'HPL' , 0 , 'number of words').mean())

print('mean words for MWS is  ' , SelectedData('author', 'MWS' , 0 , 'number of words').mean())
data['number of chars'] = data['lower text'].apply(lambda x : len(x))

print('mean chars for EAP is  ' , SelectedData('author', 'EAP' , 0 , 'number of chars').mean())

print('mean chars for HPL is  ' , SelectedData('author', 'HPL' , 0 , 'number of chars').mean())

print('mean chars for MWS is  ' , SelectedData('author', 'MWS' , 0 , 'number of chars').mean())
data['number of punctuations'] = data['lower text'].apply(lambda x : len([k for k in  x if k in r'.,;:!?|\#$%^&*/']))

print('mean punctuations for EAP is  ' , SelectedData('author', 'EAP' , 0 , 'number of punctuations').mean())

print('mean punctuations for HPL is  ' , SelectedData('author', 'HPL' , 0 , 'number of punctuations').mean())

print('mean punctuations for MWS is  ' , SelectedData('author', 'MWS' , 0 , 'number of punctuations').mean())
data['number of stop'] = data['lower text'].apply(lambda x : len([k for k in  x if k in nlp.Defaults.stop_words ]))

print('mean stop for EAP is  ' , SelectedData('author', 'EAP' , 0 , 'number of stop').mean())

print('mean stop for HPL is  ' , SelectedData('author', 'HPL' , 0 , 'number of stop').mean())

print('mean stop for MWS is  ' , SelectedData('author', 'MWS' , 0 , 'number of stop').mean())
data.head()
X = data['lower text']

y = data['author code']
VecModel = TfidfVectorizer()

XVec = VecModel.fit_transform(X)

print(f'The new shape for X is {XVec.shape}')
FeatureSelection = SelectPercentile(score_func = chi2, percentile=50)

X_data = FeatureSelection.fit_transform(XVec, y)

print('X Shape is ' , X_data.shape)
X_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.33, random_state=44, shuffle =True)

print('X_train shape is ' , X_train.shape)

print('X_test shape is ' , X_test.shape)

print('y_train shape is ' , y_train.shape)

print('y_test shape is ' , y_test.shape)
MultinomialNBModel = MultinomialNB(alpha=0.05)

MultinomialNBModel.fit(X_train, y_train)

print('MultinomialNBModel Train Score is : ' , MultinomialNBModel.score(X_train, y_train))

print('MultinomialNBModel Test Score is : ' , MultinomialNBModel.score(X_test, y_test))
data = pd.read_csv('../input/spooky-author-identification/test/test.csv')

print(f'Test data Shape is {data.shape}')

data.head()
LowerCase('text' , 'lower text')

Drop('text')

data.head()
X = data['lower text']
XVec = VecModel.transform(X)

print(f'The new shape for X is {XVec.shape}')
X_data = FeatureSelection.transform(XVec)

print('X Shape is ' , X_data.shape)
y_pred = MultinomialNBModel.predict(X_data)

y_pred_prob = MultinomialNBModel.predict_proba(X_data)

print('Predicted Value for MultinomialNBModel is : ' , y_pred[:10])

print('Prediction Probabilities Value for MultinomialNBModel is : ' , y_pred_prob[:10])
data = pd.read_csv('../input/spooky-author-identification/sample_submission/sample_submission.csv')

print(f'Test data Shape is {data.shape}')

data.head()
idd = data['id']

FinalResults = pd.DataFrame(y_pred_prob  ,columns= ['EAP','HPL','MWS'])

FinalResults.insert(0,'id',idd)
FinalResults.head()
FinalResults.to_csv("sample_submission.csv",index=False)


def next_palindrome(a):
    a = a+1
    while not is_palindrome(a):
        a += 1
    return a

def is_palindrome(a):
    return str(a) == str(a)[::-1]

if __name__ == '__main__':
    a = int(input("Enter the size of your list:\n"))
    num_list = []
    for x in range(a):
        num_list.append(int(input("Enter the number of your list:\n")))

    print(f"Your list is {num_list}")

    new_list = []
    for element in num_list:
        if element > 10:
            n = next_palindrome(element)
            new_list.append(n)

        else:
            new_list.append(element)

    print(f"output list: {new_list}")


"""
This is a basic script to output a message to twitter
User must input public and private key from Consumer
and access. They will be imported from keys file

Future Plans:
Allow more functionality allowing user input

"""

import tweepy

# Constants for secret and public keys

CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_KEY = ''
ACCESS_SECRET = ''

def post_tweet(msg):
    """
        Posts tweet onto twitter handle

        arg: accept string message msg

        returns: string message that have been inputed
    """

    # Authenticates using consumer key and secret
    auth=tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)

    # Authenticates using access key and secret
    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
    api = tweepy.API(auth)

    #print message on terminal and update status
    # to twitter handle
    print ('Tweeting' + msg)
    api.update_status(msg)

    return msg


from launch import LaunchDescription
from launch.actions import DeclareLaunchArgument, IncludeLaunchDescription
from launch.launch_description_sources import PythonLaunchDescriptionSource
from launch.substitutions import LaunchConfiguration, PathJoinSubstitution, TextSubstitution
from launch_ros.substitutions import FindPackageShare

def generate_launch_description():

    launch_args = [

        # Robot-specific configuration
        DeclareLaunchArgument('username',
                            default_value='no_value'),
        DeclareLaunchArgument('password',
                            default_value='no_value'),
        DeclareLaunchArgument('hostname',
                            default_value='no_value'),
        DeclareLaunchArgument('has_eap',
                            description="Robot includes the Extended Autonomy Package.",
                            default_value="True"),
        DeclareLaunchArgument('has_arm',
                            description='Robot includes the Spot Arm',
                            default_value="True"),

        DeclareLaunchArgument('auto_claim',
                            description='Claim ownership of the robot upon connection.',
                            default_value='False'),
        DeclareLaunchArgument('auto_power_on',
                            description='Power on the robot upon connection.',
                            default_value='False'),
        DeclareLaunchArgument('auto_stand',
                            description='Stand the robot upon connection.',
                            default_value='False')
    ]

    has_eap = LaunchConfiguration('has_eap')
    has_arm = LaunchConfiguration('has_arm')

    camera_fps = 10.0

    ## Robot bringup
    driver_include = IncludeLaunchDescription(
            PythonLaunchDescriptionSource(
                PathJoinSubstitution([FindPackageShare('spot_driver'), 'launch', 'driver.launch.py'])),
            launch_arguments=
                {'username':      LaunchConfiguration('username'),
                 'password':      LaunchConfiguration('password'),
                 'hostname':      LaunchConfiguration('hostname'),
                 'has_eap':       has_eap,
                 'has_arm':       has_arm,
                 'auto_claim':    LaunchConfiguration('auto_claim'),
                 'auto_power_on': LaunchConfiguration('auto_power_on'),
                 'auto_stand':    LaunchConfiguration('auto_stand'),
                 'camera_fps':    TextSubstitution(text=str(camera_fps))
                }.items()
            )

    # State publisher
    state_publisher_include = IncludeLaunchDescription(
      launch_description_source = PythonLaunchDescriptionSource(
        PathJoinSubstitution([
              FindPackageShare('spot_description'),
              'launch',
              'state_publisher.launch.py'
        ])
      ),
      launch_arguments={'has_arm': has_arm,
                        'has_eap': has_eap}.items()
    )

    realsense_include = IncludeLaunchDescription(
        launch_description_source = PythonLaunchDescriptionSource(
            PathJoinSubstitution([
                FindPackageShare('realsense2_camera'),
                'launch',
                'rs_launch.py'
            ])
        ),
        launch_arguments={
            'pointcloud.enable': 'True',
            'clip_distance': '2.0'
        }.items()
    )

    ## Launch
    return LaunchDescription([
        *launch_args,
        driver_include,
        state_publisher_include,
        realsense_include
    ])


from matplotlib import pyplot as plt

#nombre del archivo donde se encuentran las coordenadas de todas las particulas
dynamic_file ="../../resources/dynamic.txt"
times = 1000

def read_particles_file(name):
    list_x = []
    list_y = []
    i = 0
    with open(name) as archivo:
        for linea in archivo:
            if linea.split().__len__() == 1:
                list_x.insert(i, [])
                list_y.insert(i, [])
                i += 1
            else:
                x_aux, y_aux, vx, vy = linea.split()
                list_x[i-1].append(float(x_aux))
                list_y[i-1].append(float(y_aux))
    return list_x , list_y

if __name__ == '__main__':
    particles_x, particles_y = read_particles_file(dynamic_file)
    fig, ax = plt.subplots()
    ax.set_box_aspect(1)
    for i in range(times):
        plt.cla()
        ax.set_xlim([0, 20])
        ax.set_ylim([0, 20])
        ax.grid(which='both', color='gray', linestyle='--')
        ax.scatter(particles_x[i], particles_y[i])
        plt.title('Iteracion ' + str(i))
        plt.pause(0.02)


def findmin(arr):
    sa = 0
    sb = 0
    ra = 5
    rb = 5
    for i in range(10):
        if arr[i] == '1' and i%2 == 0:
            sa += 1
        if i%2 == 0:
            ra -= 1
        if sa > sb+rb or sb > sa+ra:
            return 10 -(ra+rb)
        if arr[i] == '1' and i%2 == 1:
            sb += 1
        if i%2 == 1:
            rb -= 1
        if sa > sb+rb or sb > sa+ra:
            return 10 -(ra+rb)
    return 10

def solve(arr):
    res = [10, 10]
    a = arr[:]
    b = arr[:]

    for i in range(0, 10, 2):
        if a[i] == '?':
            a[i] = '1'
        if b[i] == '?':
            b[i] == '0'
    for i in range(1, 9, 2):
        if a[i] == '?':
            a[i] = '0'
        if b[i] == '?':
            b[i] = '1'

    res[0] = findmin(a)
    res[1] = findmin(b)
    return res

for _ in range(int(input())):
    arr = list(input())
    print(min(solve(arr)))