import torch
import torch.nn as nn

def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    # 已看過
    # 一般卷積或是膨脹卷積
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=dilation, groups=groups, bias=False, dilation=dilation)

def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    # 已看過
    # 升維或降維使用
    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

class Bottleneck(nn.Module):
    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
    # This variant is also known as ResNet V1.5 and improves accuracy according to
    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.

    # 通過一個bottleneck在channel維度上面會加深多少倍
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
                 base_width=64, dilation=1, norm_layer=None):
        """
        :param inplanes: 輸入channel深度
        :param planes: 瓶頸結構中中間層的channel深度
        :param stride: 步距
        :param downsample: 下採樣一系列層結構
        :param groups: 組卷積數，這裡不會使用組卷積
        :param base_width: 組卷積數，這裡不會使用組卷積
        :param dilation: 膨脹係數
        :param norm_layer: 標準化層
        """
        super(Bottleneck, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        # width = planes，base_width=64且groups=1
        width = int(planes * (base_width / 64.)) * groups
        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
        # 降維
        self.conv1 = conv1x1(inplanes, width)
        self.bn1 = norm_layer(width)
        # 卷積
        self.conv2 = conv3x3(width, width, stride, groups, dilation)
        self.bn2 = norm_layer(width)
        # 升維
        self.conv3 = conv1x1(width, planes * self.expansion)
        self.bn3 = norm_layer(planes * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        # 殘差邊
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        # 已看過
        # 瓶頸結構，根據stride以及dilation會知道高和寬有沒有進行下採樣
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class ResNet(nn.Module):

    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None,
                 norm_layer=None):
        """
        :param block: 使用到的瓶頸結構
        :param layers: 每個瓶頸結構要堆疊幾層
        :param num_classes: 分類類別數
        :param zero_init_residual: 權重初始化用的
        :param groups: 組卷積，這裡不會使用到組卷積
        :param width_per_group: 組卷積，這裡不會使用到組卷積
        :param replace_stride_with_dilation: 哪幾個部分要使用到膨脹卷積
        :param norm_layer: norm_layer的實例對象，預設為None
        """
        # 已看過
        super(ResNet, self).__init__()
        # 如果沒有標準化層就創建一個BatchNorm2d
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        self._norm_layer = norm_layer

        self.inplanes = 64
        self.dilation = 1
        # 如果沒有傳入哪些要用膨脹卷積就全部設定為False
        if replace_stride_with_dilation is None:
            # each element in the tuple indicates if we should replace
            # the 2x2 stride with a dilated convolution instead
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        # 最一開始的7*7卷積，圖像會下採樣兩倍
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = norm_layer(self.inplanes)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        # 第一層殘差結構
        self.layer1 = self._make_layer(block, 64, layers[0])
        # 第二層殘差結構，不使用膨脹卷積
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
                                       dilate=replace_stride_with_dilation[0])
        # 第三層殘差結構，使用膨脹卷積
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
                                       dilate=replace_stride_with_dilation[1])
        # 第四層殘差結構，使用膨脹卷積
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
                                       dilate=replace_stride_with_dilation[2])
        # 下面這兩個都會被拿掉，因為這裡不需要做分類
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * block.expansion, num_classes)

        # 設定初始化權重
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        # Zero-initialize the last BN in each residual branch,
        # so that the residual branch starts with zeros, and each residual block behaves like an identity.
        # This improves the models by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, Bottleneck):
                    nn.init.constant_(m.bn3.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        """
        :param block: 瓶頸結構
        :param planes: 輸入channel深度
        :param blocks: 堆疊層數
        :param stride: 第一層的步距
        :param dilate: 是否使用膨脹卷積
        :return:
        """
        # 已看過
        # 構建每層殘差結構
        norm_layer = self._norm_layer
        downsample = None
        # 保留一開始的膨脹係數
        previous_dilation = self.dilation
        # 計算下次的膨脹係數
        if dilate:
            self.dilation *= stride
            stride = 1
        # 構建殘差結構的邊，進行下採樣以及channel擴維
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                norm_layer(planes * block.expansion),
            )

        layers = []
        # 第一層的殘差結構
        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
                            self.base_width, previous_dilation, norm_layer))
        self.inplanes = planes * block.expansion
        # 剩下的殘差結構
        for _ in range(1, blocks):
            layers.append(block(self.inplanes, planes, groups=self.groups,
                                base_width=self.base_width, dilation=self.dilation,
                                norm_layer=norm_layer))

        return nn.Sequential(*layers)

    def _forward_impl(self, x):
        # 已看過
        # See note [TorchScript super()]
        # 輸入 x shape [batch_size, 3, 480, 480]

        # x shape [batch_size, 64, 240, 240]，高寬下採樣兩倍
        x = self.conv1(x)
        # 標準化後接上激活函數
        x = self.bn1(x)
        x = self.relu(x)
        # x shape [batch_size, 64, 120, 120]，高寬下採樣兩倍
        x = self.maxpool(x)

        # x shape [batch_size, 256, 120, 120]
        x = self.layer1(x)
        # x shape [batch_size, 512, 60, 60]
        x = self.layer2(x)
        # x shape [batch_size, 1024, 60, 60]
        x = self.layer3(x)
        # x shape [batch_size, 2048, 60, 60]
        x = self.layer4(x)

        # 這裡會被捨棄掉，我們不會用到
        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.fc(x)

        return x

    def forward(self, x):
        # 已看過
        return self._forward_impl(x)

def _resnet(block, layers, **kwargs):
    # 已看過
    model = ResNet(block, layers, **kwargs)
    return model

def resnet50(**kwargs):
    # 已看過
    # **kwargs裡面會有說明哪幾層要使用膨脹卷積
    # 構建resnet50
    r"""ResNet-50 models from
    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_

    Args:
        pretrained (bool): If True, returns a models pre-trained on ImageNet
        progress (bool): If True, displays a progress bar of the download to stderr
    """
    # 傳入每一層要用的基礎層以及堆疊層數以及膨脹卷積要用在哪幾層
    return _resnet(Bottleneck, [3, 4, 6, 3], **kwargs)

def resnet101(**kwargs):
    # 已看過
    # **kwargs裡面會有說明哪幾層要使用膨脹卷積
    # 構建resnet101
    r"""ResNet-101 models from
    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_

    Args:
        pretrained (bool): If True, returns a models pre-trained on ImageNet
        progress (bool): If True, displays a progress bar of the download to stderr
    """
    return _resnet(Bottleneck, [3, 4, 23, 3], **kwargs)


import darklight as dl
import torch
from mobilenet_v2 import MobileNetV2
from torchvision import models

teacher=models.resnet152(pretrained=True)
student= MobileNetV2(imsize=[224, 224], inchannels=3, nclasses=1000)
dl.exportonnx(teacher, 'rn152.onnx', bsize=1, hw=[224,224])

del teacher #free up CPU or GPU memory used by teacher

dm=dl.ImageNetManager('/sfnvme/imagenet/', size=[224,224], bsize=128)

opt_params={
	'optimizer': torch.optim.AdamW,
	'okwargs': {'lr': 1e-4, 'weight_decay':0.05},
	'scheduler':torch.optim.lr_scheduler.CosineAnnealingWarmRestarts,
	'skwargs': {'T_0':10,'T_mult':2},
	'amplevel': 'O2'
	}

stream=torch.cuda.Stream()

with torch.cuda.stream(stream):
	trainer=dl.StudentTrainer(student, dm, 'rn152.onnx', opt_params=opt_params)
	trainer.train(epochs=50, save='dltest_{}.pth')


# Original code is from : https://github.com/ShangtongZhang/reinforcement-learning-an-introduction/blob/master/chapter01/tic_tac_toe.py
# This code has been rewritten and modified for testing

import numpy as np
import pickle

BOARD_ROW = 3
BOARD_COL = 3
BOARD_SPACE = BOARD_ROW*BOARD_COL

class State:
    def __init__(self):
        self.board = np.zeros((BOARD_ROW, BOARD_COL))
        self.winner = 0
        self.end = None

    def hash(self):
        h = 0
        for x in np.nditer(self.board):
            h = 3*h + x + 1
        return h

    def next(self, i, j, symbol):
        assert self.board[i, j] == 0
        new_state = State()
        new_state.board = np.copy(self.board)
        new_state.board[i, j] = symbol
        return new_state

    def is_end(self):
        if self.end is not None:
            return self.end

        check = []
        # check row
        for i in range(BOARD_ROW):
            check.append(sum(self.board[i, :]))

        # check col
        for i in range(BOARD_COL):
            check.append(sum(self.board[:, i]))

        # check diagonal
        diagonal = 0; reverse_diagonal = 0
        for i in range(BOARD_ROW):
            diagonal += self.board[i, i]
            reverse_diagonal += self.board[BOARD_ROW-i-1, i]
        check.append(diagonal)
        check.append(reverse_diagonal)

        for x in check:
            if x == 3:
                self.end = True
                self.winner = 1
                return self.end
            elif x == -3:
                self.end = True
                self.winner = -1
                return self.end

        for x in np.nditer(self.board):
            if x == 0:
                self.end = False
                return self.end

        self.end = True
        return self.end

    def print_state(self):
        for i in range(BOARD_ROW):
            print('-------------')
            out = '| '
            for j in range(BOARD_COL):
                if self.board[i, j] == 1:
                    token = '*'
                elif self.board[i, j] == -1:
                    token = 'x'
                else:
                    token = '0'
                out += token + ' | '
            print(out)
        print('-------------')

class Player:
    def __init__(self, epsilon=0, step_size=0.1):
        self.epsilon = epsilon
        self.step_size = step_size
        self.values = dict()
        self.states = []
        self.greedy = []
        self.symbol = 0

    def set_symbol(self, symbol):
        self.symbol = symbol
        for h, state in ALL_STATES.items():
            if state.is_end():
                if state.winner == symbol:
                    self.values[h] = 1.0
                elif state.winner == 0:
                    self.values[h] = 0.5
                else:
                    self.values[h] = 0.0
            else:
                self.values[h] = 0.5

    def set_state(self, state):
        self.states.append(state)
        self.greedy.append(True)

    def reset(self):
        self.states = []
        self.greedy = []

    def back_up(self):
        states = [s.hash() for s in self.states]
        for i in reversed(range(len(states)-1)):
            td = self.greedy[i]*(self.values[states[i+1]] - self.values[states[i]])
            #td = self.values[states[i+1]] - self.values[states[i]]
            self.values[states[i]] += self.step_size*td

    def play(self):
        state = self.states[-1]
        possible_moves = []
        for i in range(BOARD_ROW):
            for j in range(BOARD_COL):
                if state.board[i, j] == 0:
                    new_state_hash = state.next(i, j, self.symbol).hash()
                    possible_moves.append((self.values[new_state_hash], i, j))
        if self.epsilon > np.random.rand():
            m = possible_moves[np.random.randint(len(possible_moves))]
            self.greedy[-1] = False
            return m[1], m[2], self.symbol
        np.random.shuffle(possible_moves)
        possible_moves.sort(reverse=True, key=lambda x: x[0])
        m = possible_moves[0]
        return m[1], m[2], self.symbol

    def save_values(self):
        filepath = "first.pickle" if self.symbol==1 else "second.pickle"
        with open(filepath, 'wb') as fp:
            pickle.dump(self.values, fp, protocol=pickle.HIGHEST_PROTOCOL)

    def load_values(self):
        filepath = "first.pickle" if self.symbol==1 else "second.pickle"
        with open(filepath, 'rb') as fp:
            self.values = pickle.load(fp)

class HumanPlayer:
    def __init__(self):
        self.keys = ['q', 'w', 'e', 'a', 's', 'd', 'z', 'x', 'c']
        self.state = None
        self.symbol = 0

    def reset(self):
        self.state = None

    def set_symbol(self, symbol):
        self.symbol = symbol

    def set_state(self, state):
        self.state = state

    def play(self):
        self.state.print_state()
        key = input("Input your position: ")
        p = self.keys.index(key)
        i = p//BOARD_COL
        j = p%BOARD_COL
        return i, j, self.symbol

class Judge:
    def __init__(self, player1, player2):
        self.player1 = player1
        self.player2 = player2
        self.player1.set_symbol(1)
        self.player2.set_symbol(-1)
        self.end_state = None

    def reset(self):
        self.player1.reset()
        self.player2.reset()
        self.end_state = None

    def alternate(self):
        while True:
            yield self.player1
            yield self.player2

    def game_play(self, print_state=False):
        self.reset()
        state = State()

        if print_state:
            state.print_state()

        self.player1.set_state(state)
        self.player2.set_state(state)

        alternate = self.alternate()

        while not state.is_end():
            player = next(alternate)
            px, py, symbol = player.play()
            state = state.next(px, py, symbol)
            self.player1.set_state(state)
            self.player2.set_state(state)
            if print_state:
                state.print_state()

        self.end_state = state
        return state.winner

def explore_all_states(state, symbol, all_states):
    for i in range(BOARD_ROW):
        for j in range(BOARD_COL):
            if state.board[i, j] == 0:
                new_state = state.next(i, j, symbol)
                new_state_hash = new_state.hash()
                if new_state_hash not in all_states:
                    all_states[new_state_hash] = new_state
                    if not new_state.is_end():
                        explore_all_states(new_state, -symbol, all_states)

def get_all_states():
    all_states = dict()
    curr_state = State()
    curr_state_hash = curr_state.hash()
    all_states[curr_state_hash] = curr_state
    explore_all_states(curr_state, 1, all_states)
    return all_states

ALL_STATES = get_all_states()

def train(epochs=1_00_000):
    player1 = Player(epsilon=0.01)
    player2 = Player(epsilon=0.01)
    judge = Judge(player1, player2)

    win_player1 = 0
    win_player2 = 0
    draw = 0

    for i in range(1, epochs+1):
        winner = judge.game_play()
        player1.back_up()
        player2.back_up()
        if winner == 1:
            win_player1 += 1
        elif winner == -1:
            win_player2 += 1
        else:
            draw += 1
        if i % 10000 == 0:
            p = win_player1/(win_player1 + win_player2)
            print(f"Epoch: {i}, player1: {win_player1}, player2: {win_player2}, percetage_win_by_player1: {p}")
    player1.save_values()
    player2.save_values()

def bot_compete(epochs=1000):
    p1 = Player(epsilon=0)
    p2 = Player(epsilon=0)
    judge = Judge(p1, p2)

    p1.load_values()
    p2.load_values()

    w1 = 0; w2 = 0; d = 0
    for e in range(1, epochs+1):
        winner = judge.game_play()
        if winner == 1:
            w1 += 1
        elif winner == -1:
            w2 += 1
        else:
            d += 1
    print(f"Player1: {w1}, Player2: {w2}, Draw: {d}")

def compete(match=6):
    human = HumanPlayer()
    ai = Player(epsilon=0)

    judge = Judge(human, ai)
    ai.load_values()

    for i in range(match):
        winner = judge.game_play()
        judge.end_state.print_state()
        if winner == 1:
            print("Human wins")
        elif winner == -1:
            print("AI wins")
        else:
            print("DRAW")
    ai.save_values()

if __name__ == '__main__':
    train()
    bot_compete()
    compete(6)


#!/usr/bin/env python

# Test whether a client sends a correct UNSUBSCRIBE packet.

import inspect
import os
import socket
import sys

# From http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder
cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"..")))
if cmd_subfolder not in sys.path:
    sys.path.insert(0, cmd_subfolder)

import mosq_test

port = mosq_test.get_lib_port()

rc = 1
keepalive = 60
connect_packet = mosq_test.gen_connect("unsubscribe-test", keepalive=keepalive)
connack_packet = mosq_test.gen_connack(rc=0)

disconnect_packet = mosq_test.gen_disconnect()

mid = 1
unsubscribe_packet = mosq_test.gen_unsubscribe(mid, "unsubscribe/test")
unsuback_packet = mosq_test.gen_unsuback(mid)

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.settimeout(10)
sock.bind(('', port))
sock.listen(5)

client_args = sys.argv[1:]
env = dict(os.environ)
env['LD_LIBRARY_PATH'] = '../../lib:../../lib/cpp'
try:
    pp = env['PYTHONPATH']
except KeyError:
    pp = ''
env['PYTHONPATH'] = '../../lib/python:'+pp
client = mosq_test.start_client(filename=sys.argv[1].replace('/', '-'), cmd=client_args, env=env, port=port)

try:
    (conn, address) = sock.accept()
    conn.settimeout(10)

    if mosq_test.expect_packet(conn, "connect", connect_packet):
        conn.send(connack_packet)

        if mosq_test.expect_packet(conn, "unsubscribe", unsubscribe_packet):
            conn.send(unsuback_packet)

            if mosq_test.expect_packet(conn, "disconnect", disconnect_packet):
                rc = 0

    conn.close()
finally:
    client.terminate()
    client.wait()
    sock.close()

exit(rc)


# encoding: utf-8
"""
@author: Liurunzhi
@time: 2021/7/12 9:21
@Describe: 把视频转为图片
"""
import cv2
from cv2 import VideoWriter, VideoWriter_fourcc, imread, resize

def Video2Pic():
    videoPath = "/home/lrz/Documents/Codes/Yolov5-deepsort/video/131744.mp4"  # 读取视频路径
    imgPath = "/home/lrz/Documents/Codes/Yolov5-deepsort/video/131744/"  # 保存图片路径

    cap = cv2.VideoCapture(videoPath)
    # fps = cap.get(cv2.CAP_PROP_FPS)  # 获取帧率
    # width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))  # 获取宽度
    # height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))  # 获取高度
    suc = cap.isOpened()  # 是否成功打开
    print(suc)
    frame_count = 0
    img_count = 0
    while (suc):
        frame_count += 1
        suc, frame = cap.read()
        # if(frame_count % 2 == 0):
        print(frame)
        # cv2.imwrite(imgPath + str(frame_count).zfill(5), frame)
        cv2.imwrite(imgPath+'fish_'+str(frame_count)+'.jpg', frame)
        img_count += 1
        cv2.waitKey(1)
    print(img_count)
    cap.release()
    print("视频转图片结束！")

if __name__ == '__main__':
    Video2Pic()


def minSwaps(arr):
    ans = 0
    n=len(arr)
    temp = arr.copy()
    h = {}
    temp.sort()
    for i in range(n):
        h[arr[i]] = i
    init = 0

    for i in range(n):
        if (arr[i] != temp[i]):
            ans += 1
            init = arr[i]
            arr[i], arr[h[temp[i]]] = arr[h[temp[i]]], arr[i]
            h[init] = h[temp[i]]
            h[temp[i]] = i

    return ans
arr=[2, 8, 5, 4]
print(minSwaps(arr))


import asyncio
import random
import datetime as dt
from src.cards import MonthlyStatsCard as _Card

highlights = [
    'title_colour',
    'top_hours_colour',
    'top_hours_bg_colour',
    'top_line_colour',
    'top_date_colour',
    'top_this_colour',
    'top_last_colour',
    'top_this_hours_colour',
    'top_last_hours_colour',
    'this_month_colour',
    'last_month_colour',
    'heatmap_empty_colour',
    'weekday_background_colour',
    'weekday_colour',
    'month_background_colour',
    'month_colour',
    'stats_key_colour',
    'stats_value_colour',
    'footer_colour',
]
highlight_colour = "#E84727"
card_name = "monthly"

highlights = ['heatmap_colours']
highlight_colour = ["#E84727"]

async def get_cards():
    strings = []
    random.seed(0)
    for highlight in highlights:
        card = await _Card.generate_sample(
            skin={highlight: highlight_colour}
        )
        with open(f"../skins/spec/images/{card_name}/{highlight}.png", 'wb') as image_file:
            image_file.write(card.fp.read())

        esc_highlight = highlight.replace('_', '\\_')
        string = f"""\
\\hypertarget{{{card_name}-{highlight.replace('_', '-')}}}{{\\texttt{{{esc_highlight}}}}} & &
\\includegraphics[width=.25\\textwidth,valign=m]{{images/{card_name}/{highlight}.png}}
\\\\"""
        strings.append(string)

    print('\n'.join(strings))

if __name__ == '__main__':
    asyncio.run(get_cards())


import math
import numpy as np
import matplotlib.pyplot as plt

n_list = [1,10,50,100]

### Obtain the Y values for each X value in each function and plot the result ###

## n! (factorial)
y1 = np.array([math.factorial(n) for n in n_list])
## 2^n
y2 = np.array([2**n for n in n_list])
## n²
y3 = np.array([n**2 for n in n_list])
## sqrt(n)
y4 = np.array([math.sqrt(n) for n in n_list])
## log10(n)
y5 = np.array([math.log10(n) for n in n_list])

# Plot
plt.plot(n_list, y1, label='n!')
plt.plot(n_list, y2, label='2^n')
plt.plot(n_list, y3, label='n²')
plt.plot(n_list, y4, label='sqrt(n)')
plt.plot(n_list, y5, label='log10(n)')
plt.xlim(1, 100)
plt.ylim(1, 50)
plt.legend()
plt.show()


from rover import rover

import datetime
import numpy as np
import rospy

from sensor_msgs.msg import Imu

def thread_imu():
    print('IMU: thread starting ..')

    rospy.Subscriber('uav_imu', Imu, rover.ros_imu_callback)
    rate = rospy.Rate(100) # 100 hz

    freq = 100.0
    t = datetime.datetime.now()
    t_pre = datetime.datetime.now()
    avg_number = 100

    while not rospy.is_shutdown() and rover.on:
        t = datetime.datetime.now()
        dt = (t - t_pre).total_seconds()
        if dt < 1e-6:
            continue

        freq = (freq * (avg_number - 1) + (1 / dt)) / avg_number
        t_pre = t
        rover.freq_imu = freq

        rate.sleep()

    print('IMU: thread closed!')


#Matthew Leung
#March 2022
#Python 3

from __future__ import print_function

# import standard packages
import sys
import os
import ctypes
import time
from datetime import date
import shutil
from PIL import Image
import numpy as np

# Matrix Vision
from mvIMPACT import acquire
#from mvIMPACT.Common import exampleHelper

def printDevices(devMgr, boSilent=False, boAutomaticallyUseGenICamInterface=True):
    msg = ""
    for i in range(devMgr.deviceCount()):
        pDev = devMgr.getDevice(i)
        msg = "[" + str(i) + "]: " + pDev.serial.read() + "(" + pDev.product.read() + ", " + pDev.family.read()
        if pDev.interfaceLayout.isValid:
            if boAutomaticallyUseGenICamInterface == True:
                conditionalSetProperty(pDev.interfaceLayout, acquire.dilGenICam, boSilent)
            msg += ", interface layout: " + pDev.interfaceLayout.readS()
        if pDev.acquisitionStartStopBehaviour.isValid:
            conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser, boSilent)
            msg += ", acquisition start/stop behaviour: " + pDev.acquisitionStartStopBehaviour.readS()
        if pDev.isInUse:
            msg += ", !!!ALREADY IN USE!!!"
        print(msg + ")")

    if msg == "":
        print("No devices were found")
    return True

def supportsValue(prop, value):
    if prop.hasDict:
        validValues = []
        prop.getTranslationDictValues(validValues)
        return value in validValues

    if prop.hasMinValue and prop.getMinValue() > value:
        return False

    if prop.hasMaxValue and prop.getMaxValue() < value:
        return False

    return True

def conditionalSetProperty(prop, value, boSilent=False):
    #Modified to include boolean return
    if prop.isValid and prop.isWriteable and supportsValue(prop, value):
        prop.write(value)
        if boSilent == False:
            print("Property '" + prop.name() + "' set to '" + prop.readS() + "'.")
        return True
    return False

def getNumberFromUser():
    return int(input())

def getDeviceFromUserInput(devMgr, boSilent=False, boAutomaticallyUseGenICamInterface=True):
    for i in range(devMgr.deviceCount()):
        pDev = devMgr.getDevice(i)
        msg = "[" + str(i) + "]: " + pDev.serial.read() + "(" + pDev.product.read() + ", " + pDev.family.read()
        if pDev.interfaceLayout.isValid:
            if boAutomaticallyUseGenICamInterface == True:
                conditionalSetProperty(pDev.interfaceLayout, acquire.dilGenICam, boSilent)
            msg += ", interface layout: " + pDev.interfaceLayout.readS()
        if pDev.acquisitionStartStopBehaviour.isValid:
            conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser, boSilent)
            msg += ", acquisition start/stop behaviour: " + pDev.acquisitionStartStopBehaviour.readS()
        if pDev.isInUse:
            msg += ", !!!ALREADY IN USE!!!"
        print(msg + ")")

    print("Please enter the number in front of the listed device followed by [ENTER] to open it: ", end='')
    devNr = getNumberFromUser()
    if (devNr < 0) or (devNr >= devMgr.deviceCount()):
        print("Invalid selection!")
        return None
    return devMgr.getDevice(devNr)

def requestENTERFromUser():
    msg = "Press Enter to continue..."
    input(msg)

# Start the acquisition manually if this was requested(this is to prepare the driver for data capture and tell the device to start streaming data)
def manuallyStartAcquisitionIfNeeded(pDev, fi):
    if pDev.acquisitionStartStopBehaviour.read() == acquire.assbUser:
        result = fi.acquisitionStart()
        if result != acquire.DMR_NO_ERROR:
            print("'FunctionInterface.acquisitionStart' returned with an unexpected result: " + str(result) + "(" + acquire.ImpactAcquireException.getErrorCodeAsString(result) + ")")

# Stop the acquisition manually if this was requested
def manuallyStopAcquisitionIfNeeded(pDev, fi):
    if pDev.acquisitionStartStopBehaviour.read() == acquire.assbUser:
        result = fi.acquisitionStop()
        if result != acquire.DMR_NO_ERROR:
            print("'FunctionInterface.acquisitionStop' returned with an unexpected result: " + str(result) + "(" + acquire.ImpactAcquireException.getErrorCodeAsString(result) + ")")

#######################################################################################

def read_uint12(data_chunk):
    data = np.frombuffer(data_chunk, dtype=np.uint8)
    fst_uint8, mid_uint8, lst_uint8 = np.reshape(data, (data.shape[0] // 3, 3)).astype(np.uint16).T
    fst_uint12 = (fst_uint8 << 4) + (mid_uint8 >> 4)
    snd_uint12 = ((mid_uint8 % 16) << 8) + lst_uint8
    return np.reshape(np.concatenate((fst_uint12[:, None], snd_uint12[:, None]), axis=1), 2 * fst_uint12.shape[0])

def acquire_frame(filename, exptime=0.0, ampgain=0.0, verbosity=0):
    #if self._verbose: self._log("Image acqisition start")
    #exptime = min(max(exptime, 0.00001), 20.0)
    devMgr = acquire.DeviceManager()
    printDevices(devMgr)
    pDev = devMgr.getDevice(0)

    #In order to use acquire.CameraSettingsBlueCOUGAR, you CANNOT use the
    #GenICam interface! I.e. the following line is not allowed:
    #conditionalSetProperty(pDev.interfaceLayout, acquire.dilGenICam) #Use GenICam
    #Technically, it doesn't really matter if you have this line^
    #What matters is you set to device specific below!

    #IMPORTANT: It is required to have one of the two lines below!
    #They are the same thing; conditionalSetProperty does .write on pDev.interfaceLayout
    #Here we set interfaceLayout to device specific instead of GenICam
    conditionalSetProperty(pDev.interfaceLayout, acquire.dilDeviceSpecific)
    pDev.interfaceLayout.write(acquire.dilDeviceSpecific)

    conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser)
    pDev.open()
    fi = acquire.FunctionInterface(pDev)
    result = fi.acquisitionStop()

    #ac = acquire.AcquisitionControl(pDev)
    #ac.pixelFormat.write(acquire.idpfMono12)

    #pDev.interfaceLayout.getTranslationDictString() #this does nothing but read the a dictionary containing the valid values for this property, for default key of 0
    interfaceLayout_str_value = pDev.interfaceLayout.readS() #the value of the interfaceLayout property, in str format
    interfaceLayout_value = pDev.interfaceLayout.read()
    interfaceLayout_str_value_from_dict = pDev.interfaceLayout.getTranslationDictString(interfaceLayout_value)
    print("The current interface layout is "+interfaceLayout_str_value)
    print("From reading the dictionary, the current interface layout is "+interfaceLayout_str_value_from_dict)

    ac = acquire.CameraSettingsBlueCOUGAR(pDev)
    ac.pixelFormat.write(acquire.idpfMono12)
    ac.expose_us.write(800)
    ac.gain_dB.write(0)
    ac.offset_pc.write(0)
    #ifc = acquire.ImageFormatControl(pDev , "Base")
    #ifc.pixelFormat.write("Mono12")

    #csbd = acquire.CameraSettingsBlueDevice(pDev)
    #csbd = acquire.CameraSettingsBlueCOUGAR(pDev)
    #csbd.expose_us.write(int(exptime * 1000000))
    #csbd.autoControlMode.write(0)
    #csbd.autoGainControl.write(0)
    #csbd.gain_dB.write(float(ampgain)) #Electronic gain in dB
    #csbd.pixelFormat.write(acquire.idpfMono12)
    #csbd.offset_pc.write(float(offset_pc)) #A float property defining the analogue sensor offset in percent of the allowed range
    #csbd.offset_pc.write(float(10))
    #conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser)
    result = fi.acquisitionStart()
    while fi.imageRequestSingle() == acquire.DMR_NO_ERROR:
        if verbosity > 0: print(".",end='')
    if verbosity > 0: print('')
    pPreviousRequest = None
    manuallyStartAcquisitionIfNeeded(pDev, fi)
    requestNr = fi.imageRequestWaitFor(-1)
    if fi.isRequestNrValid(requestNr):
        pRequest = fi.getRequest(requestNr)
        if pRequest.isOK:
            pRequest.imageData.read()

            #cbuf = (ctypes.c_char * pRequest.imageSize.read()).from_address(int(pRequest.imageData.read()))
            #channelType = numpy.uint16
            #arr = numpy.frombuffer(cbuf, dtype = numpy.uint16)
            #arr.shape = (pRequest.imageHeight.read(), pRequest.imageWidth.read(), pRequest.imageChannelCount.read())
            #pixel_data = Image.fromarray(arr[:,:,0])
            #print(pixel_data)

            cbuf = (ctypes.c_char * pRequest.imageSize.read()).from_address(int(pRequest.imageData.read()))
            if pRequest.imageChannelBitDepth.read() > 8:
                channelType = np.uint16
            else:
                channelType = np.uint8
                print("WARNING: Camera is acquiring in 8 bit!")

            total_size = pRequest.imageHeight.read() * pRequest.imageWidth.read() * pRequest.imageChannelCount.read()
            print(total_size)

            arr = np.frombuffer(cbuf, dtype=channelType) #Only works for not 12 bit <-- Not true?
            #if arr.size < total_size:
            #    arr = read_uint12(cbuf) #NEED THIS FOR 12 BIT BUFFER!
            #else:
            #    print("WARNING: Camera is acquiring in 8 bit!")
            arr_rs = arr.reshape(pRequest.imageHeight.read(), pRequest.imageWidth.read(), pRequest.imageChannelCount.read())

            print("The number of unique values in the image is {}".format(np.size(np.unique(arr_rs))))

            image = arr_rs
            import matplotlib.pyplot as plt
            plt.figure()
            plt.imshow(image, cmap='gray')
            plt.colorbar()
            plt.show()

            plt.figure()
            plt.hist(image.ravel(), 2**12)
            plt.title('Histogram')
            plt.xlabel('Pixel Value')
            plt.ylabel('Frequency')
            plt.show()

        if pPreviousRequest != None:
            pPreviousRequest.unlock()
        pPreviousRequest = pRequest
        #fi.imageRequestSingle()
        result = fi.acquisitionStop()
    else:
        print("imageRequestWaitFor failed (" + str(requestNr) + ", " +
              acquire.ImpactAcquireException.getErrorCodeAsString(requestNr) + ")")
    result = fi.acquisitionStop()
    pDev.close()
    return True

if __name__ == "__main__":
    acquire_frame(None)

# Saved for now:
# def __acquireExposure(self, exptime, fname):
#         #print("__acquireExposure: ",exptime,fname)
#         verbose = 0
#         exptime = min(max(exptime, 0.00001), 2.0)
#         devMgr = acquire.DeviceManager()
#         pDev = devMgr.getDevice(0)
#         conditionalSetProperty(pDev.interfaceLayout, acquire.dilGenICam)
#         conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser)
#         validValues = []
#         prop.getTranslationDictValues(validValues)
#         # #if pDev.isInUse:
#         # #    print(", !!!ALREADY IN USE!!!", end='')
#         pDev.open()
#         fi = acquire.FunctionInterface(pDev)
#         result = fi.acquisitionStop()
#         csbd = acquire.CameraSettingsBlueDevice(pDev)
#         csbd.expose_us.write(int(exptime * 1000000))
#         csbd.autoControlMode.write(0)
#         csbd.autoGainControl.write(0)
#         csbd.gain_dB.write(float(self.__gain))
#         csbd.pixelFormat.write(acquire.idpfMono12)
#         csbd.offset_pc.write(float(self.__percentOffset))
#         #dio = acquire.DigitalIOControl(pDev)
#         print(pDev.AnalogControl)
#         conditionalSetProperty(pDev.acquisitionStartStopBehaviour, acquire.assbUser)
#         #print("Turning off auto black level")
#         #ac.blackLevelAuto.write(0)
#         result = fi.acquisitionStart()
#         while fi.imageRequestSingle() == acquire.DMR_NO_ERROR:
#             if verbose: print(".",end='')
#         if verbose: print('')
#         pPreviousRequest = None
#         manuallyStartAcquisitionIfNeeded(pDev, fi)
#         requestNr = fi.imageRequestWaitFor(-1)
#         #fname = 'NULL'
#         if fi.isRequestNrValid(requestNr):
#             #print("RequestNrValid")
#             pRequest = fi.getRequest(requestNr)
#             if pRequest.isOK():
#                 #print("isOK")
#                 cbuf = (ctypes.c_char * pRequest.imageSize.read()).from_address(long(pRequest.imageData.read()))
#                 channelType = numpy.uint16 if pRequest.imageChannelBitDepth.read() > 8 else numpy.uint8
#                 arr = numpy.frombuffer(cbuf, dtype = channelType)
#                 arr.shape = (pRequest.imageHeight.read(), pRequest.imageWidth.read(), pRequest.imageChannelCount.read())
#                 img = Image.fromarray(arr[:,:,0])
#                 hdu = fits.PrimaryHDU(img)
#                 hdul = fits.HDUList([hdu])
#         #         #fname = dataDir + '/'+ prefix + '_' + filebase + '.fits'
#                 #print("Writing file: ",fname)
#                 hdul.writeto(fname,overwrite=True)
#             if pPreviousRequest != None:
#                 pPreviousRequest.unlock()
#             pPreviousRequest = pRequest
#             #fi.imageRequestSingle()
#             result = fi.acquisitionStop()
#         else:
#             print("imageRequestWaitFor failed (" + str(requestNr) + ", " + ImpactAcquireException.getErrorCodeAsString(requestNr) + ")")
#         result = fi.acquisitionStop()
#         pDev.close()
#         return


from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer
from cleaner import clean_corpus

CORPUS_FILE = "chat.txt"
# Creating ChatBot Instance
chatbot = ChatBot(
    'Rome',
    storage_adapter='chatterbot.storage.SQLStorageAdapter',
    database_uri='sqlite:///database.sqlite3'

    )

trainer = ListTrainer(chatbot)
cleaned_corpus = clean_corpus(CORPUS_FILE)
trainer.train(cleaned_corpus)

exit_conditions = (":q", "quit", "exit")
while True:
    query = input("YOU: ")
    if query in exit_conditions:
        break
    else:
        print(f"{chatbot.get_response(query)}")


#strategy to solve the problem:
    #why:
        #using slow and fast strategy to find middle. slow.next will be head of the second half
            #slow behind and move one while fast is first and move twice. start with slow, fast = head, head.next
        #reverse the second half. remember to change from slow.next = None
        #we do need temp1, temp2 for merge two linked list together. because we we need to update first and second for next iteration

    #variables:
        #slow and fast (ListNode): to find middle node

    #error note:
        #condition to stop slow, fast strategy
            #while fast and fast.next
        #condition for merge. because with slow and fast pointer second half is always smaller than first half
            #while second:
        #rember to complete the first half after find middle
            #slow.next = None
        #we do need temp1, temp2 for merge two linked list together. because we we need to update first and second for next iteration

class Solution:
    def reorderList(self, head: ListNode) -> None:
        # find middle
        slow, fast = head, head.next
        while fast and fast.next:
            slow = slow.next
            fast = fast.next.next

        # reverse second half
        second = slow.next
        prev = slow.next = None
        while second:
            tmp = second.next
            second.next = prev
            prev = second
            second = tmp

        # merge two halfs
        first, second = head, prev
        while second:
            tmp1, tmp2 = first.next, second.next
            first.next = second
            second.next = tmp1
            first, second = tmp1, tmp2


import nltk,sys
import numpy as np,pandas as pd
import matplotlib.pyplot as plt
import nltk.data
import csv
from nltk.stem.porter import *
pd.options.mode.chained_assignment = None

data = pd.read_csv("mbti_1.csv")
shortdata=data.iloc[:,-1]
shortdata=shortdata.head()
shortdata1=data.iloc[0:5,0]

print(shortdata)

#removing stopwords
from nltk.corpus import stopwords
stop=stopwords.words("english")
print('------Removing stopwords------')
shortdata=shortdata.apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
print(shortdata)

#stemming of words
ps = PorterStemmer()
print('-------Stemming--------')
shortdata = shortdata.apply(lambda x: ' '.join([ps.stem(word) for word in x.split() ]))
print(shortdata)

#removing non-alphabets
shortdata=shortdata.apply(lambda x: ' '.join([word for word in x.split() if word.isalpha()]))

from nltk.stem.wordnet import WordNetLemmatizer
lmtzr = WordNetLemmatizer()
#print(shortdata)
print('-------Lemmatization--------')
shortdata = shortdata.apply(lambda x: ' '.join([lmtzr.lemmatize(word,'v') for word in x.split() ]))
print(shortdata)

print('--------Removing punctuations--------')
def clear_punctuation(s):
	import string
	#print("\n")
	clear_string = ""
	for symbol in s:
		if symbol not in string.punctuation:
			clear_string += symbol
	return clear_string

shortdata = shortdata.apply(lambda x: ''.join(clear_punctuation(x))  )
for line in shortdata:
	print(line)
	print('\n')

def strip_all_entities(text):
	import string
	entity_prefixes = ['@']
	for separator in  string.punctuation:
		if separator not in entity_prefixes :
			text = text.replace(separator,' ')
	words = []
	for word in text.split():
		word = word.strip()
		if word:
			if word[0] not in entity_prefixes:
				words.append(word)
	return ' '.join(words)

shortdata = shortdata.apply(lambda x: ''.join(strip_all_entities(x))  )
print(shortdata)

with open("tweets.csv","w") as file:
	for i in range(5):
		file.write(shortdata1[i])
		file.write("     ")
		file.write(shortdata[i])
		file.write('\n')


# -*- coding:utf-8 -*-

import json
from Utility import Utils
from .models import *

class ProductType:
    unknown = -1
    stock = 0
    fund = 1
    index = 2

class ProductHelper:

    @staticmethod
    def __getAllStocks(dataprovider):
        result = dataprovider.getAllStockProdInfo()
        stocks = json.loads(result)
        return stocks

    @staticmethod
    def __getAllFunds(dataprovider):
        result = dataprovider.getAllFundProdInfo()
        funds = json.loads(result)
        return funds

#    ***********     refresh product performance test result     ***********
# 2016-01-25 22:49:49.581000 Start refreshing product DB...
# 2016-01-25 22:49:49.583000 Deleting product DB...
# 2016-01-25 22:49:49.676000 Start query product info...
# 2016-01-25 22:49:50.804000 Done.
# 2016-01-25 22:49:51.130000 Saved to DB.
    @staticmethod
    def refresh_productDB(dataprovider):
        Utils.log("Start refreshing product DB...")

        Utils.log("Deleting product DB...")
        ProductInfo.objects.all().delete()

        Utils.log("Start query product info...")
        stocks = ProductHelper.__getAllStocks(dataprovider)
        funds = ProductHelper.__getAllFunds(dataprovider)

        Utils.log("Saving to DB....")
        products = []
        # insert product info
        for stock in stocks:
            product = ProductInfo(symbol=stock["symbol"], cn_name=stock["name"], issue_type="stock")
            products.append(product)
        for fund in funds:
            product = ProductInfo(symbol=fund["symbol"], cn_name=fund["name"], issue_type="fund")
            products.append(product)
        ProductInfo.objects.bulk_create(products)
        Utils.log("Done.")

        message = "Fetching stock info done:" + str(len(products))
        return message


#!/usr/bin/env python3
from transports import get_transport, TransportError

def SNMP_audit():
    connection = get_transport('SNMP')
    try:
        sysDescr = connection.get_snmpdata('.1.3.6.1.2.1.1.1.0')
        interfacesQuantity = connection.get_snmpdata('.1.3.6.1.2.1.2.1.0')[0]
        listOfInterfaces = []
        for interface in range(1, interfacesQuantity):
            listOfInterfaces.append([
                ((connection.get_snmpdata('.1.3.6.1.2.1.2.2.1.2.' + str(interface))[0]).prettyPrint()),
                (connection.get_snmpdata('.1.3.6.1.2.1.2.2.1.7.' + str(interface))[0].prettyPrint())
            ])
    except Exception:
        return None

    listOfInterfaces.sort()

    SNMP_audit_info = {
        'sysDescr': sysDescr[0],
        'listOfInterfaces': listOfInterfaces
    }
    return SNMP_audit_info

def SSH_audit():
    try:
        connection = get_transport('SSH')
    except TransportError:
        print('Warning: SSH service is unavailable')
        return None
    SSH_audit_info = {
        'sys_info': connection.exec('uname -a'),
        'cpu_info': connection.exec('lscpu'),
        'kernel_version': connection.exec('cat /proc/version'),
        'sys_users': connection.exec('cat /etc/passwd'),
        'ip_macs': connection.exec('ip l'),
        'packages': connection.exec('apt list --installed')
    }
    return SSH_audit_info

def retrieve_audit_info():
    print('Getting SNMP and SSH system info...')
    SNMP = SNMP_audit()
    SSH = SSH_audit()
    if ((SNMP == None) and (SSH == None)):
        return None
    else:
        return [SNMP, SSH]


import types

class Config(dict):

    def __init__(self, defaults=None):
        dict.__init__(self, defaults or {})

    def from_file(self, filename):
        config = types.ModuleType('config')
        config.__file__ = filename
        try:
            with open(filename, mode='rb') as config_file:
                exec(compile(config_file.read(), filename, 'exec'), config.__dict__)
        except IOError as e:
            e.strerror = 'Не удается загрузить файл конфигурации {}'.format(e.strerror)
            raise
        self.set_config(config)

    def set_config(self, config):
        for key in dir(config):
            if key.isupper():
                self[key] = getattr(config, key)


from hog_cpp.fhog import fhog
import numpy as np

'''
https://github.com/lawpdas/fhog-python
'''

def get_hog(img):
    M = np.zeros(img.shape[:2], dtype='float32')
    O = np.zeros(img.shape[:2], dtype='float32')
    H = np.zeros([img.shape[0] // 4, img.shape[1] // 4, 32], dtype='float32')  # python3
    fhog.gradientMag(img.astype(np.float32), M, O)
    fhog.gradientHist(M, O, H)
    H = H[:, :, :31]

    return H

'''
if __name__ == "__main__":
    img_path = 'D:/DATASET/OTB100/Basketball/img/0001.jpg'
    img = cv2.imread(img_path)

    sub = img[0:40, 0:40]

    H = get_hog(sub)
    print(H)
'''


from typing import Type, Optional, Dict, Any

from django import template
from django.forms import Form
from django.urls import reverse_lazy

register = template.Library()

@register.inclusion_tag('portfolio/components/action-btn.html')
def action_btn(btn_txt: str, id_prefix: str, style: str,
            section: Optional[bool] = True,
            url: Optional[str] = None,
            pk: Optional[int] = None,
            form: Optional[str] = None) -> Dict[str, Any]:
    args = [pk, ] if pk else None
    href = reverse_lazy(url, args=args) if url else '#'
    section = 'section' if section else ''
    return {
        'btn_txt': btn_txt,
        'id_prefix': id_prefix,
        'style': style,
        'section': section,
        'href': href,
        'pk': pk,
        'form': form,
    }


import torch
import torch.nn as nn

class AMFNet(nn.Module):

    def __init__(self, input_size, hidden):
        super(AMFNet, self).__init__()

        self.fc1 = nn.Linear(input_size, hidden)  # 5*5 from image dimension
        self.output_layer = nn.Linear(hidden, 1)

    def forward(self, x):
        fc1_z = self.fc1(x) #[batch_size, seq_len, hidden_dim]
        fc1_output = torch.relu(fc1_z)

        output_layer_z = self.output_layer(fc1_output)

        return output_layer_z


from node import *

def calc_dof(nodes):
    count = 0
    for i in nodes:
        i.degrees[0] = count
        count += 1
        i.degrees[1] = count
        count += 1

# def calc_dof(nodes):
#     buffer = []
#     count = 0
#     for i in nodes:
#         print(i.id_number)
#       # i.restriction[0] : restriçao do nó em x(tem se for 1)
#       # i.restriction[1] : restrição do nó em y(tem se for 2)
#         if(i.restrictions[0] == 1):
#             buffer.append([i, i.restrictions[0], 'x'])
#         else:
#             i.degrees[0] = count
#             count += 1
#         if (i.restrictions[1] == 1):
#             buffer.append([i, i.restrictions[1], 'y'])
#         else:
#             i.degrees[1] = count
#             count += 1
#     for e in buffer:
#         if(e[2] == 'x'):
#             e[0].degrees[0] = count
#             count += 1
#         if(e[2] == 'y'):
#             e[0].degrees[1] = count
#             count += 1


import numpy as np
from sklearn.metrics import roc_curve

def determine_edges(scores, nbins, indicesQ=False, score_min=-1, score_max=1):
    if nbins > len(scores):
        print('More bins than values. Attempting to return bins with one value each.')
        return determine_edges(scores, len(scores), indicesQ=indicesQ, score_min=score_min, score_max=score_max)
    scores_sorted = scores.copy()
    scores_index = scores.argsort()
    scores_sorted = scores_sorted[scores_index]
    aux = np.linspace(0, len(scores_sorted) - 1, nbins + 1).astype(int) + 1
    bin_indices = np.zeros(len(scores_sorted)).astype(int)
    bin_indices[:aux[1]] = 0
    for i in range(1, len(aux) - 1):
        bin_indices[aux[i]:aux[i + 1]] = i
    bin_edges = np.zeros(nbins + 1)
    for i in range(0, nbins - 1):
        bin_edges[i + 1] = np.mean(np.concatenate((
            scores_sorted[bin_indices == i][scores_sorted[bin_indices == i] == max(scores_sorted[bin_indices == i])],
            scores_sorted[bin_indices == (i + 1)][
                scores_sorted[bin_indices == (i + 1)] == min(scores_sorted[bin_indices == (i + 1)])])))
    bin_edges[0] = score_min
    bin_edges[-1] = score_max
    bin_indices = bin_indices[np.argsort(scores_index)]
    if len(np.unique(bin_edges)) < len(bin_edges):
        return determine_edges(scores, nbins - 1, indicesQ=indicesQ, score_min=score_min, score_max=score_max)
    else:
        if indicesQ:
            return bin_edges, bin_indices
        else:
            return bin_edges

def bin_confidences_and_accuracies(confidences, ground_truth, bin_edges, indices):
    i = np.arange(0, bin_edges.size-1)
    aux = indices == i.reshape((-1, 1))
    counts = aux.sum(axis=1)
    weights = counts / np.sum(counts)
    correct = np.logical_and(aux, ground_truth).sum(axis=1)
    a = np.repeat(confidences.reshape(1, -1), bin_edges.size-1, axis=0)
    a[np.logical_not(aux)] = 0
    bin_accuracy = correct / counts
    bin_confidence = a.sum(axis=1) / counts
    return weights, bin_accuracy, bin_confidence

def get_ks(confidences, ground_truth):
    n = len(ground_truth)
    order_sort = np.argsort(confidences)
    ks = np.max(np.abs(np.cumsum(confidences[order_sort])/n-np.cumsum(ground_truth[order_sort])/n))
    return ks

def get_brier(confidences, ground_truth):
    # Compute Brier Score
    brier = np.zeros(confidences.shape)
    brier[ground_truth] = (1-confidences[ground_truth])**2
    brier[np.logical_not(ground_truth)] = (confidences[np.logical_not(ground_truth)])**2
    brier = np.mean(brier)
    return brier

def get_ece(confidences, ground_truth, nbins):
    # Repeated code from determine edges. Here it is okay if the bin edges are not uniquely defined
    confidences_sorted = confidences.copy()
    confidences_index = confidences.argsort()
    confidences_sorted = confidences_sorted[confidences_index]
    aux = np.linspace(0, len(confidences_sorted) - 1, nbins + 1).astype(int) + 1
    bin_indices = np.zeros(len(confidences_sorted)).astype(int)
    bin_indices[:aux[1]] = 0
    for i in range(1, len(aux) - 1):
        bin_indices[aux[i]:aux[i + 1]] = i
    bin_edges = np.zeros(nbins + 1)
    for i in range(0, nbins - 1):
        bin_edges[i + 1] = np.mean(np.concatenate((
            confidences_sorted[bin_indices == i][confidences_sorted[bin_indices == i] == max(confidences_sorted[bin_indices == i])],
            confidences_sorted[bin_indices == (i + 1)][
                confidences_sorted[bin_indices == (i + 1)] == min(confidences_sorted[bin_indices == (i + 1)])])))
    bin_edges[0] = 0
    bin_edges[-1] = 1
    bin_indices = bin_indices[np.argsort(confidences_index)]

    weights, bin_accuracy, bin_confidence = bin_confidences_and_accuracies(confidences, ground_truth, bin_edges,
                                                                           bin_indices)
    ece = np.dot(weights, np.abs(bin_confidence - bin_accuracy))
    return ece

def compute_scores(confidences, ground_truth, nbins):

    # Compute ECE
    ece = get_ece(confidences, ground_truth, nbins)

    # Compute Brier
    brier = get_brier(confidences, ground_truth)

    # Compute KS
    ks = get_ks(confidences, ground_truth)

    return ece, ks, brier

def get_binary_clf_stats(ground_truth, scores):
    thr = np.unique(scores.copy())
    # false positives
    fp = np.sum(scores[np.logical_not(ground_truth)] >= thr.reshape(-1, 1), axis=1)
    # true negative
    tn = np.sum(scores[np.logical_not(ground_truth)] < thr.reshape(-1, 1), axis=1)
    # false negative
    fn = np.sum(scores[ground_truth] < thr.reshape(-1, 1), axis=1)
    # true positive
    tp = np.sum(scores[ground_truth] >= thr.reshape(-1, 1), axis=1)
    fpr = fp / (fp + tn)
    tpr = tp / (tp + fn)

    return fpr, tpr, thr


"""
比对主力连续合约数据
"""
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt

origin_data = pd.read_pickle("/data/废弃数据/continuous_main_contract_series.pkl")
data1 = pd.read_pickle("/data/continuous_contract_series/continuous_main_contract_series 1.pkl")
data2 = pd.read_pickle("/data/continuous_contract_series/continuous_main_contract_series 3.pkl")
data3 = pd.read_pickle("/data/continuous_contract_series/continuous_main_contract_series 5.pkl")

symbol_list = origin_data.underlying_symbol.unique().tolist()

for symbol in tqdm(symbol_list):
    series0 = origin_data[origin_data.underlying_symbol==symbol].set_index("datetime")['continuous_close']
    series1 = data1[data1.underlying_symbol==symbol].set_index("datetime")['continuous_close']
    series2 = data2[data2.underlying_symbol==symbol].set_index("datetime")['continuous_close']
    series3 = data3[data3.underlying_symbol==symbol].set_index("datetime")['continuous_close']
    plt.figure(figsize=(20, 8))
    series0.plot(figsize=(20, 8), label="series0")
    series1.plot(figsize=(20, 8), label="series1")
    series2.plot(figsize=(20, 8), label="series2")
    series3.plot(figsize=(20, 8), label="series3")
    plt.grid()
    plt.legend()
    plt.title(symbol)
    plt.savefig(f"D:/LFProjects/NewPythonProject/结果输出/主力连续合约对比/{symbol}.png")
    plt.show()


import sys
import os
import gc
import math
import numpy as np
import cv2
import glob
import imageio
from PIL import Image
from random import choice
from keras.utils import Sequence
import matplotlib.pyplot as plt
from keras import backend as K
from keras.layers import Lambda, Input
from keras.models import Model
import tensorflow as tf
from subprocess import Popen, PIPE
from timeit import default_timer as timer
from losses import psnr2 as psnr

class Metrics():
    def __init__(self,use_gpu=True):
        '''
        net - ['squeeze','alex','vgg']
        '''

    def psnr_video(self,videopath1, videopath2):
        cap1 = cv2.VideoCapture(videopath1)
        cap2 = cv2.VideoCapture(videopath2)
        psnr_values = []
        while(cap1.isOpened() and cap2.isOpened()):
            ret1, frame1 = cap1.read()
            ret2, frame2 = cap2.read()
            if(ret1 and ret2):
                psnr_values.append(self.calculate_psnr(frame1, frame2))
            else:
                break
        cap1.release()
        cap2.release()
        return np.mean(psnr_values)

    def ssim_video(self,videopath1, videopath2):
        cap1 = cv2.VideoCapture(videopath1)
        cap2 = cv2.VideoCapture(videopath2)
        ssim_values = []
        while(cap1.isOpened() and cap2.isOpened()):
            ret1, frame1 = cap1.read()
            ret2, frame2 = cap2.read()
            if(ret1 and ret2):
                ssim_values.append(self.calculate_ssim(frame1, frame2))
            else:
                break
        cap1.release()
        cap2.release()
        return np.mean(ssim_values)

    # Calculate psnr between two images
    def calculate_psnr(self,img1, img2):
        # img1 and img2 have range [0, 255]
        img1 = img1.astype(np.float64)
        img2 = img2.astype(np.float64)
        mse = np.mean((img1 - img2)**2)
        if mse == 0:
            return float('inf')
        return 20 * math.log10(255.0 / math.sqrt(mse))

    # Aux to Calculate psnr between two images
    def ssim(self,img1, img2):
        C1 = (0.01 * 255)**2
        C2 = (0.03 * 255)**2

        img1 = img1.astype(np.float64)
        img2 = img2.astype(np.float64)
        kernel = cv2.getGaussianKernel(11, 1.5)
        window = np.outer(kernel, kernel.transpose())

        mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]  # valid
        mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
        mu1_sq = mu1**2
        mu2_sq = mu2**2
        mu1_mu2 = mu1 * mu2
        sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
        sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
        sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

        ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                                (sigma1_sq + sigma2_sq + C2))
        return ssim_map.mean()

    # Calculate ssim between two images
    def calculate_ssim(self,img1, img2):
        '''calculate SSIM
        the same outputs as MATLAB's
        img1, img2: [0, 255]
        '''
        if not img1.shape == img2.shape:
            raise ValueError('Input images must have the same dimensions.')
        if img1.ndim == 2:
            return self.ssim(img1, img2)
        elif img1.ndim == 3:
            if img1.shape[2] == 3:
                ssims = []
                for i in range(3):
                    ssims.append(self.ssim(img1, img2))
                return np.array(ssims).mean()
            elif img1.shape[2] == 1:
                return self.ssim(np.squeeze(img1), np.squeeze(img2))
        else:
            raise ValueError('Wrong input image dimensions.')

class DataLoader(Sequence):
    def __init__(self, datapath, batch_size, height_hr, width_hr, scale, crops_per_image, media_type,time_step):
        """
        :param string datapath: filepath to training images
        :param int height_hr: Height of high-resolution images
        :param int width_hr: Width of high-resolution images
        :param int height_hr: Height of low-resolution images
        :param int width_hr: Width of low-resolution images
        :param int scale: Upscaling factor
        """

        # Store the datapath
        self.datapath = datapath
        self.batch_size = batch_size
        self.height_hr = height_hr
        self.height_lr = int(height_hr / scale)
        self.width_hr = width_hr
        self.width_lr = int(width_hr / scale)
        self.channel = 1
        self.scale = scale
        self.crops_per_image = crops_per_image
        self.media_type  = media_type
        self.time_step=time_step
        self.total_imgs = None

        # Options for resizing
        self.options = [Image.NEAREST, Image.BILINEAR, Image.BICUBIC, Image.LANCZOS]

        # Check data source
        self.img_paths = []

        if os.path.isdir(self.datapath):
            self.get_paths()

    def get_paths(self):
        for dirpath, _, filenames in os.walk(self.datapath):
            for filename in [f for f in filenames if any(filetype in f.lower() for filetype in ['jpeg', 'png', 'jpg','mp4','264','webm','wma'])]:
                self.img_paths.append(os.path.join(dirpath, filename))
        self.total_imgs = len(self.img_paths)
        print(">> Found {} images in dataset".format(self.total_imgs))

    def random_crop(self, img, random_crop_size):
        # Note: image_data_format is 'channel_last'
        assert img.shape[2] == 3
        height, width = img.shape[0], img.shape[1]
        dy, dx = random_crop_size
        x = np.random.randint(0, width - dx + 1)
        y = np.random.randint(0, height - dy + 1)
        return img[y:(y+dy), x:(x+dx), :]

    def fix_crop(self, img, dy, dx, y, x):
        return img[y:(y+dy), x:(x+dx), :]

    @staticmethod
    def scale_lr_imgs(imgs):
        """Scale low-res images prior to passing to SRGAN"""
        return imgs / 255.

    @staticmethod
    def unscale_lr_imgs(imgs):
        """Un-Scale low-res images"""
        return imgs * 255

    @staticmethod
    def scale_hr_imgs(imgs):
        """Scale high-res images prior to passing to SRGAN"""
        return imgs / 127.5 - 1

    @staticmethod
    def unscale_hr_imgs(imgs):
        """Un-Scale high-res images"""
        return (imgs + 1.) * 127.5

    def count_frames_manual(self,cap):
        count=0
        while(cap.isOpened()):
            ret, frame = cap.read()
            if(ret):
                count +=1
            else:
                break
        return count

    def count_frames(self,cap):
        '''Count total frames in video'''
        try:
            total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
        except:
            total = self.count_frames_manual(cap)
        return total

    def get_random_frames(self,n_fms=1,cap=None,time_step=1):
        """Get random number of video frames"""
        np.random.seed(int(1000000*(timer()%1)))
        t_frames = self.count_frames(cap)
        if(time_step==1):
            choiced_frames = np.random.randint(t_frames, size=n_fms)
        else:
            choiced_frames = np.random.randint(t_frames-time_step, size=n_fms)
        return choiced_frames

    @staticmethod
    def load_img(path):
        img = Image.open(path)
        if img.mode != 'RGB':
            img = img.convert('RGB')
        return np.array(img)

    def load_frame(self,videopath,time_step=1,colorspace='YCbCr'):
        """Get n_imgs random frames from the video"""
        cap = cv2.VideoCapture(videopath)
        if(not cap.isOpened()):
            print("Error to open video: ",videopath)
            return -1
        choiced_frame=self.get_random_frames(1,cap,time_step)
        cap.set(1,choiced_frame)
        frames = []
        for i in list(range(time_step)):
            ret, frame = cap.read()
            if not ret:
                print(">> Erro to access frames.")
                return -1
            if colorspace=='YCbCr':
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)[:,:,:self.channel]
                frames.append(frame)
            else:
                frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
        cap.release()
        return np.array(frames)

    def resize(self,shape,scale,image):

        # 3 channel images of arbitrary shape
        inp = Input(shape=shape)
        try:
            out = Lambda(lambda image: K.tf.image.resize_bicubic(image, (shape[0]//scale, shape[1]//scale)))(inp)
        except :
            # if you have older version of tensorflow
            out = Lambda(lambda image: K.tf.image.resize_bicubic(image, shape[0]//scale, shape[1]//scale))(inp)
        model = Model(inputs=inp, outputs=out)
        out = model.predict(image[np.newaxis, ...])
        return np.squeeze(out, axis=0).astype(np.uint8)

    def __len__(self):
        return int(self.total_imgs / float(self.batch_size))

    def __getitem__(self, idx):
        return self.load_batch(idx=idx)

    def load_batch(self,idx=0, img_paths=None, training=True, bicubic=False):
        """ Loads a batch of images or video"""
        if(self.media_type=='i'):
            #print("1. Media type image folder")
            imgs_lr, imgs_hr = self.load_batch_image(idx, img_paths=img_paths, training=training,bicubic=False)
        elif(self.media_type=='v' and os.path.isdir(self.datapath)):
            #print("2. Media type video folder")
            imgs_lr, imgs_hr = self.load_batch_video(idx, img_paths=None, training=training, bicubic=False)
        elif(self.media_type=='v' and not os.path.isdir(self.datapath)):
            #print("3. Media type single video")
            imgs_lr, imgs_hr = self.load_batch_video(idx, img_paths=self.datapath, training=training, bicubic=False)
        elif(self.media_type=='vt' and os.path.isdir(self.datapath)):
            #print("4. Media type single video")
            imgs_lr, imgs_hr = self.load_temporal_batch_video(idx, img_paths=None, training=training, bicubic=False)
        else:
            print("Error: type of media fail")
        return imgs_lr, imgs_hr

    def load_temporal_batch_video(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of frames from videos folder"""
        # Starting index to look in
        cur_idx=np.random.randint(self.total_imgs)
        if not img_paths:
            cur_idx = idx*self.batch_size
        #print('cur_idx:',cur_idx)
        if training == False:
            cur_idx = idx

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        for i in range(self.time_step):
            imgs_lr.append([])
        while True:
            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break
            try:
                # Load image
                ori_imgs_hr = None
                if img_paths:
                    ori_imgs_hr = self.load_frame(img_paths[cur_idx],self.time_step)
                else:
                    ori_imgs_hr = self.load_frame(self.img_paths[cur_idx],self.time_step)

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        assert ori_imgs_hr.shape[3] == self.channel
                        height, width = ori_imgs_hr.shape[1], ori_imgs_hr.shape[2]
                        x = np.random.randint(0, width - self.width_hr + 1)
                        y = np.random.randint(0, height - self.height_hr + 1)
                        t_step_img_crops = []
                        for img_hr in ori_imgs_hr:
                            t_step_img_crops.append(self.fix_crop(img_hr,self.height_hr,self.width_hr,y,x))
                        img_crops.append(t_step_img_crops)
                else:
                    assert ori_imgs_hr.shape[3] == self.channel
                    t_step_img_crops = []
                    for img_hr in ori_imgs_hr:
                       t_step_img_crops.append(img_hr)
                    img_crops.append(t_step_img_crops)

                # Downscale the HR images and save
                for t_step_img_hr in img_crops:
                    t_step_imgs_lr = []
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break
                    i=0
                    for img_hr in t_step_img_hr:
                        # For LR, do bicubic downsampling
                        method = Image.BICUBIC if bicubic else choice(self.options)
                        lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                        img_lr = Image.fromarray(img_hr.astype(np.uint8))
                        img_lr = np.array(img_lr.resize(lr_shape, method))
                        # Scale color values
                        img_hr = self.scale_hr_imgs(img_hr)
                        img_lr = self.scale_lr_imgs(img_lr)

                        # Store LR images
                        imgs_lr[i].append(img_lr)
                        i+=1
                    # Store LR and HR images
                    imgs_hr.append(np.expand_dims(img_hr, axis=0))
            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        for i in range(self.time_step):
            imgs_lr[i] = np.array(imgs_lr[i])
        imgs_hr = np.array(imgs_hr)
        imgs_hr=imgs_hr.reshape(self.batch_size,img_hr.shape[0],img_hr.shape[1],self.channel)

        # Return image batch
        return imgs_lr, imgs_hr

    def load_batch_video(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of frames from videos folder"""
        # Starting index to look in
        cur_idx = 0
        if not img_paths:
            cur_idx = idx*self.batch_size
        #print('cur_idx:',cur_idx)

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        while True:

            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break

            try:
                # Load image
                img_hr = None
                if img_paths:
                    #print('1. video: ',img_paths[cur_idx])
                    img_hr = self.load_frame(img_paths[cur_idx])
                    #print(img_hr.shape)
                else:
                    #print('2. video: ',self.img_paths[cur_idx])
                    img_hr = self.load_frame(self.img_paths[cur_idx])
                    #print(img_hr.shape)

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        #print(idx, cur_idx, "Loading crop: ", i)
                        img_crops.append(self.random_crop(img_hr, (self.height_hr, self.width_hr)))
                else:
                    img_crops = [img_hr]

                # Downscale the HR images and save
                for img_hr in img_crops:

                    # TODO: Refactor this so it does not occur multiple times
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break

                    # For LR, do bicubic downsampling
                    method = Image.BICUBIC if bicubic else choice(self.options)
                    lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                    img_lr = Image.fromarray(img_hr.astype(np.uint8))
                    img_lr = np.array(img_lr.resize(lr_shape, method))

                    # Scale color values
                    img_hr = self.scale_hr_imgs(img_hr)
                    img_lr = self.scale_lr_imgs(img_lr)

                    # Store images
                    imgs_hr.append(img_hr)
                    imgs_lr.append(img_lr)

            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        if training:
            imgs_hr = np.array(imgs_hr)
            imgs_lr = np.array(imgs_lr)

        # Return image batch
        return imgs_lr, imgs_hr

    def load_batch_image(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of images from datapath folder"""

        # Starting index to look in
        cur_idx = 0

        if not img_paths:
            cur_idx = idx*self.batch_size

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        while True:

            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break

            try:
                # Load image
                img_hr = None
                if img_paths:
                    img_hr = self.load_img(img_paths[cur_idx])
                else:
                    img_hr = self.load_img(self.img_paths[cur_idx])

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        #print(idx, cur_idx, "Loading crop: ", i)
                        img_crops.append(self.random_crop(img_hr, (self.height_hr, self.width_hr)))
                else:
                    img_crops = [img_hr]

                # Downscale the HR images and save
                for img_hr in img_crops:

                    # TODO: Refactor this so it does not occur multiple times
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break

                    # For LR, do bicubic downsampling
                    method = Image.BICUBIC if bicubic else choice(self.options)
                    lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                    img_lr = Image.fromarray(img_hr.astype(np.uint8))
                    img_lr = np.array(img_lr.resize(lr_shape, method))

                    # Scale color values
                    img_hr = self.scale_hr_imgs(img_hr)
                    img_lr = self.scale_lr_imgs(img_lr)

                    # Store images
                    imgs_hr.append(img_hr)
                    imgs_lr.append(img_lr)

            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        if training:
            imgs_hr = np.array(imgs_hr)
            imgs_lr = np.array(imgs_lr)

        # Return image batch
        return imgs_lr, imgs_hr

    def sr_genarator(self,model,img_lr):
        """Predict sr frame given a LR frame"""
        # Predict high-resolution version (add batch dimension to image)
        img_lr=self.scale_lr_imgs(img_lr)
        img_sr = model.generator.predict(np.expand_dims(img_lr, 0))
        # Remove batch dimension
        img_sr = img_sr.reshape(img_sr.shape[1], img_sr.shape[2], img_sr.shape[3])
        img_sr = self.unscale_hr_imgs(img_sr)
        return img_sr

    def write_sr_images(self,model, input_images, output_images):
        """
        :param SRGAN model: The trained SRGAN model
        :param list input_images: List of filepaths for input images
        :param list output_images: List of filepaths for output images
        """

        # Load the images to perform test on images
        imgs_lr, imgs_hr = self.load_batch(idx=0,img_paths=input_images, training=False)
        # Scale color values
        imgs_hr = self.unscale_hr_imgs(np.array(imgs_hr))
        imgs_lr = self.unscale_lr_imgs(np.array(imgs_lr))

        # Create super resolution images
        imgs_sr = []
        time_elapsed = []
        for img_lr,img_hr in zip(imgs_lr,imgs_hr):
            start = timer()
            img_sr = self.sr_genarator(model,img_lr)
            end = timer()
            time_elapsed.append(end - start)

            img_sr = Image.fromarray(img_sr.astype(np.uint8))
            img_sr.save(output_images.split(".")[0]+"SR.png")
            #imageio.imwrite(output_images.split(".")[0]+"SR.png", img_sr)

            img_hr = Image.fromarray(img_hr.astype(np.uint8))
            img_hr.save(output_images.split(".")[0]+"HR.png")
            #imageio.imwrite(output_images.split(".")[0]+"HR.png", img_hr)

            img_lr = Image.fromarray(img_lr.astype(np.uint8))
            img_lr.save(output_images.split(".")[0]+"LR.png")
            #imageio.imwrite(output_images.split(".")[0]+"LR.png", img_lr)
        return time_elapsed

class VideoRestore():
    #def __init__(self):

    @staticmethod
    def scale_lr_imgs(imgs):
        """Scale low-res images prior to passing to SRGAN"""
        return imgs / 255.

    @staticmethod
    def unscale_hr_imgs(imgs):
        """Un-Scale high-res images"""
        return (imgs + 1.) * 127.5

    def count_frames_manual(self,cap):
        count=0
        while(cap.isOpened()):
            ret, frame = cap.read()
            if(ret):
                count +=1
            else:
                break
        return count

    def count_frames(self,cap):
        '''Count total frames in video'''
        try:
            total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
        except:
            total = self.count_frames_manual(cap)
        return total

    def sr_genarator(self,model,img_lr):
        """Predict sr frame given a LR frame"""
        # Predict high-resolution version (add batch dimension to image)
        img_sr = np.squeeze(
                    model.generator.predict(img_lr,
                        batch_size=1
                    ),
                    axis=0
                )
        # Remove batch dimension
        img_sr = self.unscale_hr_imgs(img_sr)
        return img_sr

    def write_srvideo(self, model=None,lr_videopath=None,sr_videopath=None,print_frequency=30,crf=15):
        """Predict SR video given LR video """
        cap = cv2.VideoCapture(lr_videopath)
        if cap.isOpened():
            fps = math.ceil(cap.get(cv2.CAP_PROP_FPS))
            # ffmpeg setup '-qscale', '5',
            p = Popen(['ffmpeg', '-y', '-f', 'image2pipe', '-vcodec', 'mjpeg', '-r', str(fps), '-i', '-', '-vcodec', 'libx264','-preset', 'veryslow', '-crf',str(crf), '-r', str(fps), sr_videopath], stdin=PIPE)
        else:
            print("Error to open low resolution video")
            return -1

        # Get video total frames
        t_frames = self.count_frames(cap)
        #cria arquivo video hr if hr video is open
        count = 0
        time_elapsed = []
        while cap.isOpened():
            ret, frame = cap.read()
            if ret:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                start = timer()
                img_sr = self.sr_genarator(model,frame)
                end = timer()
                time_elapsed.append(end - start)
                im = Image.fromarray(img_sr.astype(np.uint8))
                im.save(p.stdin, 'JPEG')
                count +=1
            else:
                break
            if(count % print_frequency == 0):
                print('Time per Frame: '+str(np.mean(time_elapsed))+'s')
                print('Estimated time: '+str(np.mean(time_elapsed)*(t_frames-count)/60.)+'min')
        p.stdin.close()
        p.wait()
        cap.release()
        return time_elapsed

    def write_temporal_srvideo(self, model=None,lr_videopath=None,sr_videopath=None,print_frequency=30,crf=15,time_step=1):
        """Predict SR video given LR video """
        cap = cv2.VideoCapture(lr_videopath)
        if cap.isOpened():
            fps = math.ceil(cap.get(cv2.CAP_PROP_FPS))
            # ffmpeg setup '-qscale', '5',
            p = Popen(['ffmpeg', '-y', '-f', 'image2pipe', '-vcodec', 'mjpeg', '-r', str(fps), '-i', '-', '-vcodec', 'libx264','-preset', 'veryslow', '-crf',str(crf), '-r', str(fps), sr_videopath], stdin=PIPE)
        else:
            print("Error to open low resolution video")
            return -1

        # Get video total frames
        t_frames = self.count_frames(cap)
        #cria arquivo video hr if hr video is open
        count = 0
        time_elapsed = []
        frame_steps = []
        cap.set(1,0)
        while cap.isOpened() and count < t_frames:
            ret, frame = cap.read()
            if ret:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                if count > 0:
                    frame_steps.append(np.expand_dims(self.scale_lr_imgs(frame),0))
                    frame_steps.pop(0)
                else:
                    frame_steps = [np.expand_dims(self.scale_lr_imgs(frame),0) for f in range(time_step)]
                start = timer()
                img_sr = self.sr_genarator(model,frame_steps)
                end = timer()
                time_elapsed.append(end - start)
                im = Image.fromarray(img_sr.astype(np.uint8))
                im.save(p.stdin, 'JPEG')
                count +=1
            else:
                break
            if(count % print_frequency == 0):
                print('Time per Frame: '+str(np.mean(time_elapsed))+'s')
                print('Estimated time: '+str(np.mean(time_elapsed)*(t_frames-count)/60.)+'min')
        p.stdin.close()
        p.wait()
        cap.release()
        return time_elapsed

def plot_test_images(model, loader, datapath_test, test_output, epoch, name='SRGAN'):
    """
    :param SRGAN model: The trained SRGAN model
    :param DataLoader loader: Instance of DataLoader for loading images
    :param str datapath_test: path to folder with testing images
    :param string test_output: Directory path for outputting testing images
    :param int epoch: Identifier for how long the model has been trained
    """

    try:
        # Get the location of test images
        test_images = [os.path.join(datapath_test, f) for f in os.listdir(datapath_test) if any(filetype in f.lower() for filetype in ['jpeg','mp4','264', 'png', 'jpg'])]

        # Load the images to perform test on images
        imgs_lr, imgs_hr = loader.load_batch(img_paths=test_images, training=False, bicubic=True)

        # Create super resolution and bicubic interpolation images
        imgs_sr = []
        imgs_bc = []
        for i in range(len(test_images)):

            # Bicubic interpolation
            pil_img = loader.unscale_lr_imgs(imgs_lr[i]).astype('uint8')
            pil_img = Image.fromarray(pil_img)
            hr_shape = (imgs_hr[i].shape[1], imgs_hr[i].shape[0])

            imgs_bc.append(
                loader.scale_lr_imgs(
                    np.array(pil_img.resize(hr_shape, resample=Image.BICUBIC))
                )
            )

            # SRGAN prediction
            imgs_sr.append(
                np.squeeze(
                    model.predict(
                        np.expand_dims(imgs_lr[i], 0),
                        batch_size=1
                    ),
                    axis=0
                )
            )

        # Unscale colors values
        imgs_lr = [loader.unscale_lr_imgs(img).astype(np.uint8) for img in imgs_lr]
        imgs_hr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_hr]
        imgs_sr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_sr]

        # Loop through images
        for img_hr, img_lr, img_bc, img_sr, img_path in zip(imgs_hr, imgs_lr, imgs_bc, imgs_sr, test_images):

            # Get the filename
            filename = os.path.basename(img_path).split(".")[0]

            # Images and titles
            images = {
                'Low Resolution': img_lr,
                'Bicubic Interpolation': img_bc,
                name: img_sr,
                'Original': img_hr
            }

        # Plot the images. Note: rescaling and using squeeze since we are getting batches of size 1
            fig, axes = plt.subplots(1, 4, figsize=(40, 10))
            for i, (title, img) in enumerate(images.items()):
                axes[i].imshow(img)
                axes[i].set_title("{} - {}".format(title, img.shape))
                axes[i].axis('off')
            plt.suptitle('{} - Epoch: {}'.format(filename, epoch))

            # Save directory
            savefile = os.path.join(test_output, "{}-Epoch{}.png".format(filename, epoch))
            fig.savefig(savefile)
            plt.close()
            gc.collect()
    except Exception as e:
        print(e)

def plot_temporal_test_images(model, loader, datapath_test, test_output, epoch, name='TSRGAN',time_step=3):
    """
    :param SRGAN model: The trained SRGAN model
    :param DataLoader loader: Instance of DataLoader for loading images
    :param str datapath_test: path to folder with testing images
    :param string test_output: Directory path for outputting testing images
    :param int epoch: Identifier for how long the model has been trained
    """

    try:
        # Get the location of test images
        test_images = [os.path.join(datapath_test, f) for f in os.listdir(datapath_test) if any(filetype in f.lower() for filetype in ['mp4','webm','264'])]

        # Create super resolution and bicubic interpolation images
        imgs_sr = []
        imgs_bc = []
        _imgs_lr = []
        _imgs_hr = []
        for i in range(len(test_images)):
            # Load the images to perform test on images
            imgs_lr, imgs_hr = loader.load_batch(idx=i,training=True, bicubic=True)

            # Bicubic interpolation
            pil_img = loader.unscale_lr_imgs(np.squeeze(imgs_lr[time_step-1], axis=0)).astype('uint8')
            pil_img = Image.fromarray(pil_img)
            hr_shape = (imgs_hr.shape[2], imgs_hr.shape[1])

            _imgs_lr.append(np.squeeze(imgs_lr[time_step-1], axis=0))
            _imgs_hr.append(np.squeeze(imgs_hr, axis=0))
            imgs_bc.append(
                loader.scale_lr_imgs(
                    np.array(pil_img.resize(hr_shape, resample=Image.BICUBIC))
                )
            )

            # TSRGAN prediction
            imgs_sr.append(
                np.squeeze(
                    model.generator.predict(imgs_lr,
                        batch_size=1
                    ),
                    axis=0
                )
            )

        # Unscale colors values
        imgs_lr = [loader.unscale_lr_imgs(img).astype(np.uint8) for img in _imgs_lr]
        imgs_hr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in _imgs_hr]
        imgs_sr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_sr]

        # Loop through images
        for img_hr, img_lr, img_bc, img_sr, img_path in zip(imgs_hr, imgs_lr, imgs_bc, imgs_sr, test_images):

            # Get the filename
            filename = os.path.basename(img_path).split(".")[0]

            # Images and titles
            images = {
                'Low Resolution': [img_lr, img_hr],
                'Bicubic Interpolation': [img_bc,img_hr],
                name: [img_sr,img_hr],
                'Original': [img_hr,None]
            }

            # Plot the images. Note: rescaling and using squeeze since we are getting batches of size 1
            fig, axes = plt.subplots(1, 4, figsize=(40, 10))
            for i, (title, imgs) in enumerate(images.items()):
                axes[i].imshow(imgs[0])
                axes[i].set_title("{} - {} {}".format(title, imgs[0].shape, ("- psnr: "+str(round(psnr(imgs[1],imgs[0]),2)) if (title == name or title == 'Bicubic Interpolation') else " ")))
                axes[i].axis('off')
            plt.suptitle('{} - Epoch: {}'.format(filename, epoch))

            # Save directory
            savefile = os.path.join(test_output, "{}-Epoch{}.png".format(filename, epoch))
            fig.savefig(savefile)
            plt.close()
            gc.collect()
    except Exception as e:
        print(e)

def PSNR(y_true, y_pred):
        """
        PSNR is Peek Signal to Noise Ratio, see https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio

        The equation is:
        PSNR = 20 * log10(MAX_I) - 10 * log10(MSE)

        Since input is scaled from -1 to 1, MAX_I = 1, and thus 20 * log10(1) = 0. Only the last part of the equation is therefore neccesary.
        """
        return -10.0 * K.log(K.mean(K.square(y_pred - y_true))) / K.log(10.0)


#!/usr/bin/env python3

import sys
sys.path.append('../..')

import numpy as np

from neml.cp import polycrystal, crystallography, slipharden, sliprules, inelasticity, kinematics, singlecrystal, polefigures, postprocessors
from neml.math import rotations, tensors, nemlmath, matrix
from neml import elasticity

import matplotlib.pyplot as plt

if __name__ == "__main__":
  # Number of crystals and number of threads
  N = 1000
  nthreads = 3

  # Strain direction, rate, and number of steps
  L = np.array([[0.0,0,0],[0,1.0,0],[0,0,-1.0]])
  erate = 1.0e-4
  steps = 100
  emax = 2.0

  # Temperature in K
  T = 0

  # Model
  a = 2.9511*0.1 # nm
  c = 4.68433*0.1 # nm

  # Elastic constants in MPa
  C11 = 160000.0
  C33 = 181000.0
  C44 = 46500.0
  C12 = 90000.0
  C13 = 66000.0

  # Constant part of the strength for slip and twin
  tau0 = np.array([170.0]*3+[90.5]*3+[210]*6+[180.0]*6+[250.0]*6)/10.0

  # Not realistic but I just want it to roll quickly
  # Hardening coefficients for slip (H1) and twinning (H2)
  H1 = 10.0
  H2 = 10.0

  # Reference slip rate and rate sensitivity exponent
  g0 = 1.0
  n = 12.0

  # Twin threshold
  twin_threshold = 0.75

  # Sets up the interaction matrix
  M = matrix.SquareMatrix(24, type = "diagonal_blocks",
      data = [H1,H2], blocks = [12,12])

  print(M)

  # Sets up the lattice crystallography
  lattice = crystallography.HCPLattice(a, c)
  # Basal
  lattice.add_slip_system([1,1,-2,0],[0,0,0,1])
  # Prismatic
  lattice.add_slip_system([1,1,-2,0],[1,0,-1,0])
  # Pyramidal <c+a>
  lattice.add_slip_system([1,1,-2,-3],[1,1,-2,2])
  # Tension twinning
  lattice.add_twin_system([-1,0,1,1],[1,0,-1,2],[1,0,-1,1],[1,0,-1,-2])
  # Compression twinning
  lattice.add_twin_system([1,1,-2,-3],[1,1,-2,2],[2,2,-4,3],[1,1,-2,-4])

  # Sets up the actual deformation tensor
  L *= erate
  dt = emax / steps / erate
  # Randomly selects initial orientations
  orientations = rotations.random_orientations(N)

  # Plots an initial basal pole figure
  polefigures.pole_figure_discrete(orientations,[0,0,0,1],lattice)
  plt.title("Initial, <0001>")
  plt.show()

  # Sets up the linear elastic tensor
  emodel = elasticity.TransverseIsotropicLinearElasticModel(
      C11,C33,C12,C13,C44,"components")

  # Sets up the slip system strength model (this is what you'll change)
  strength = slipharden.SimpleLinearHardening(M, tau0)
  # Sets up the slip rule
  slipmodel = sliprules.PowerLawSlipRule(strength, g0, n)
  # Sets up the model inelastic rate kinematics
  imodel = inelasticity.AsaroInelasticity(slipmodel)
  # Sets up the overall model kinematics
  kmodel = kinematics.StandardKinematicModel(emodel, imodel)

  # This is the object that causes twins to recrystallize
  twinner = postprocessors.PTRTwinReorientation(twin_threshold)

  # Sets up the single crystal model
  model = singlecrystal.SingleCrystalModel(kmodel, lattice,
      postprocessors = [], verbose = False, linesearch = True,
      miter = 100, max_divide = 10)

  # Sets up the poly crystal model
  pmodel = polycrystal.TaylorModel(model, orientations, nthreads = nthreads)

  # Runs the exmaple
  h_n = pmodel.init_store()

  d_inc = nemlmath.sym(0.5*(L+L.T))
  w_inc = nemlmath.skew(0.5*(L-L.T))

  s_n = np.zeros((6,))
  d_n = np.zeros((6,))
  w_n = np.zeros((3,))

  u_n = 0.0
  p_n = 0.0

  for i in range(steps):
    print(i)
    d_np1 = d_n + d_inc * dt
    w_np1 = w_n + w_inc * dt
    s_np1, h_np1, A_np1, B_np1, u_np1, p_np1 = pmodel.update_ld_inc(
        d_np1, d_n, w_np1, w_n, T, T, dt, 0,
        s_n, h_n, u_n, p_n)

    d_n = np.copy(d_np1)
    w_n = np.copy(w_np1)

    s_n = np.copy(s_np1)
    h_n = np.copy(h_np1)

    u_n = u_np1
    p_n = p_np1

  # Plots a second, as-rolled basal pole figure
  polefigures.pole_figure_discrete(pmodel.orientations(h_np1),
      [0,0,0,1],lattice)
  plt.title("Final, <0001>")
  plt.show()