matrizA = []
matrizB = []
lista = []
matrizC = []

def adicao(matriz1, matriz2):
    global lista, matrizC
    matrizC = []
    for i in range(0, 2):
        for j in range(0,2):
            x = matriz1[i][j] + matriz2[i][j]
            lista.append(x)
        matrizC.append(lista)
        lista = []
    return matrizC

def subtracaoAB(matriz1, matriz2):
    global lista, matrizC
    matrizC = []
    for i in range(0, 2):
        for j in range(0,2):
            x = matriz1[i][j] - matriz2[i][j]
            lista.append(x)
        matrizC.append(lista)
        lista = []
    return matrizC

def subtracaoBA(matriz1, matriz2):
    global lista, matrizC
    matrizC = []
    for i in range(0, 2):
        for j in range(0,2):
            x = matriz1[i][j] - matriz2[i][j]
            lista.append(x)
        matrizC.append(lista)
        lista = []
    return matrizC

def multiplicaAB(matriz1, matriz2):
    global lista, matrizC
    matrizC = []
    lista = []
    a = (matriz1[0][0] * matriz2[0][0]) + (matriz1[0][1] * matriz2[1][0])
    b = (matriz1[0][0] * matriz2[0][1]) + (matriz1[0][1] * matriz2[1][1])
    lista.append(a)
    lista.append(b)
    matrizC.append(lista)
    lista = []

    c = (matriz1[1][0] * matriz2[0][0]) + (matriz1[1][1] * matriz2[1][0])
    d = (matriz1[1][0] * matriz2[0][1]) + (matriz1[1][1] * matriz2[1][1])
    lista.append(c)
    lista.append(d)
    matrizC.append(lista)
    lista = []

    return matrizC

def multiplicaBA(matriz1, matriz2):
    global lista, matrizC
    matrizC =[]
    lista = []
    a = (matriz1[0][0] * matriz2[0][0]) + (matriz1[0][1] * matriz2[1][0])
    b = (matriz1[0][0] * matriz2[0][1]) + (matriz1[0][1] * matriz2[1][1])
    lista.append(a)
    lista.append(b)
    matrizC.append(lista)
    lista = []

    c = (matriz1[1][0] * matriz2[0][0]) + (matriz1[1][1] * matriz2[1][0])
    d = (matriz1[1][0] * matriz2[0][1]) + (matriz1[1][1] * matriz2[1][1])
    lista.append(c)
    lista.append(d)
    matrizC.append(lista)
    lista = []

    return matrizC

def identidade():
    matrizC = [[1, 0], [0, 1]]
    return matrizC

def inversa():
    print(f'Desculpa, só consegui fazer até aqui')

def resposta(matriz):
    for linha in matriz:
        for elemento in linha:
            print(f'{elemento} ', end='')
        print()

# Matriz A
for i in range(0, 2):
    for j in range(0,2):
        x = int(input(f'Adiciona o elemento {i+1}x{j+1} da matriz A: '))
        lista.append(x)
    matrizA.append(lista)
    lista = []

# Matriz B
print()
for i in range(0, 2):
    for j in range(0,2):
        x = int(input(f'Adiciona o elemento {i+1}x{j+1} da matriz B: '))
        lista.append(x)
    matrizB.append(lista)
    lista = []

# Printa matriz A
for linha in matrizA:
    for elemento in linha:
        print(f'{elemento} ', end='')
    print()

# Printa matriz B
print()
for linha in matrizB:
    for elemento in linha:
        print(f'{elemento} ', end='')
    print()

# Menu de opções
while True:
    print('''
    Menu de Opções
1 = Adição de Matrizes
2 = Subtração de A por B
3 = Subtração de B por A
4 = Multiplicação de A por B
5 = Multiplicação de B por A
6 = Matriz identidade
    ''', end='')
    opcao = int(input('O que deseja fazer: '))
    print()

    if opcao == 1:
        # Adição
        adicao(matrizA, matrizB)
        print(f'A soma das matrizes deu: ')
        resposta(matrizC)

    elif opcao == 2:
        # Subtração A por B
        subtracaoAB(matrizA, matrizB)
        print(f'A subtração da matriz A pela matriz B deu: ')
        resposta(matrizC)

    elif opcao == 3:
        # Subtração B por A
        subtracaoBA(matrizB, matrizA)
        print(f'A subtração da matriz B pela matriz A deu: ')
        resposta(matrizC)

    elif opcao == 4:
        # Multiplicação A por B
        multiplicaAB(matrizA, matrizB)
        print(f'A multiplicação da matriz A pela matriz B deu: ')
        resposta(matrizC)

    elif opcao == 5:
        # Multiplicação B por A
        multiplicaBA(matrizB, matrizA)
        print()
        print(f'A multiplicação da matriz B pela matriz A deu: ')
        resposta(matrizC)

    elif opcao == 6:
        # Matriz identidade
        identidade()
        print()
        print(f'A matriz identidade é: ')
        resposta(matrizC)

    elif opcao == 7:
        # Matriz inversa
        inversa()
    else:
        print('Opção inválida! Tente novamente')
        continue

    sair = str(input('Deseja Sair: [S/N]')).strip().upper()[0]
    if sair == 'S':
        break
print('Programa Encerrado!')


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

''' 还没写好
KindEditor是一套开源的HTML可视化编辑器
    Kindeditor扫描类:
        Kindeditor 目录遍历
            CVE-2018-18950
                Payload: https://baizesec.github.io/bylibrary/%E6%BC%8F%E6%B4%9E%E5%BA%93/02-%E7%BC%96%E8%BE%91%E5%99%A8%E6%BC%8F%E6%B4%9E/Kindeditor/KindEditor%203.4.2%263.5.5%E5%88%97%E7%9B%AE%E5%BD%95%E6%BC%8F%E6%B4%9E/

file:///etc/passwd
file:///C:\Windows\System32\drivers\etc\hosts
'''

from lib.api.dns import dns
from lib.initial.config import config
from lib.tool.md5 import md5, random_md5
from lib.tool.logger import logger
from lib.tool.thread import thread
from lib.tool import check
from thirdparty import requests
from time import sleep

class Kindeditor():
    def __init__(self):
        self.timeout = config.get('timeout')
        self.headers = config.get('headers')
        self.proxies = config.get('proxies')

        self.app_name = 'Kindeditor'
        self.md = md5(self.app_name)
        self.cmd = 'echo ' + self.md

        self.cve_2018_18950_payloads = [
            {
                'path': 'php/file_manager_json.php?path=/',
                'data': ''
            },
        ]

    def cve_2018_18950_scan(self, url):
        ''' KindEditor 3.4.2/3.5.5版本中的php/file_manager_json.php文件存在目录遍历漏洞,
            远程攻击者可借助"path"参数利用该漏洞浏览文件
        '''
        vul_info = {}
        vul_info['app_name'] = self.app_name
        vul_info['vul_type'] = 'FIle-Read'
        vul_info['vul_id'] = 'CVE-2018-18950'
        vul_info['vul_method'] = 'GET'
        vul_info['headers'] = {}

        # headers = self.headers.copy()
        # headers.update(vul_info['headers'])

        for payload in self.cve_2018_18950_payloads:
            path = payload['path']
            data = payload['data']
            target = url + path

            vul_info['path'] = path
            vul_info['data'] = data
            vul_info['target'] = target

            try:
                res = requests.get(
                    target,
                    timeout=self.timeout,
                    headers=self.headers,
                    data=data,
                    proxies=self.proxies,
                    verify=False,
                    allow_redirects=False
                )
                logger.logging(vul_info, res.status_code, res)                        # * LOG
            except requests.ConnectTimeout:
                logger.logging(vul_info, 'Timeout')
                return None
            except requests.ConnectionError:
                logger.logging(vul_info, 'Faild')
                return None
            except:
                logger.logging(vul_info, 'Error')
                return None

            if ('11'):
                results = {
                    'Target': target,
                    'Type': [vul_info['app_name'], vul_info['vul_type'], vul_info['vul_id']],
                    'Method': vul_info['vul_method'],
                    'Payload': {
                        'Url': url,
                        'Path': path
                    }
                }
                return results

    def addscan(self, url, vuln=None):
        if vuln:
            return eval('thread(target=self.{}_scan, url="{}")'.format(vuln, url))

        return [
            thread(target=self.cve_2018_18950_scan, url=url)
        ]

kindeditor = Kindeditor()


from collections import defaultdict
from heapq import *
from isp_rest import LinkManager, IntentManager
from isp_utility import intent_p2p_install

def dijkstra(edges, source, destination):  # Dijkstra Shortest Path Algorithm
    graph = defaultdict(list)
    for node_src, node_dst, bw in edges:
        graph[node_src].append((bw, node_dst))

    # dist records the min value of each node in heap.
    queue, seen, distance = [(0, source, ())], set(), {source: 0}
    while queue:
        (cost, vertex1, path) = heappop(queue)
        if vertex1 in seen:
            continue

        seen.add(vertex1)
        path += (vertex1,)
        if vertex1 == destination:
            return cost, path

        for bw, vertex2 in graph.get(vertex1, ()):
            if vertex2 in seen:
                continue

            # Not every edge will be calculated. The edge which can improve the value of node in heap will be useful.
            if vertex2 not in distance or cost + bw < distance[vertex2]:
                distance[vertex2] = cost + bw
                heappush(queue, (distance[vertex2], vertex2, path))

    return float("inf")

def generate_edge():    # Generate Primary Topology
    link = LinkManager()
    link_json, src_dev, src_port, dst_dev, dst_port, link_bw = link.get_link()
    ref_bw = 100000000
    cost = [ref_bw // int(link_bw[x]) for x in range(len(link_bw))]
    edges = list(zip(src_dev, dst_dev, cost))
    device_in = list(zip(src_dev, src_port))
    device_eg = list(zip(dst_dev, dst_port))
    edges_with_port = list(zip(device_in, device_eg, cost))
    return edges, device_in, device_eg, edges_with_port

def calculate_dijkstra_path(src, dst):  # Calculate Primary Best Path from Primary Topology
    edge, dev_in, dev_eg, edge_port = generate_edge()
    di_dijkstra = []
    de_dijkstra = []
    pi_dijkstra = []
    pe_dijkstra = []
    di = []
    de = []
    pi = []
    pe = []

    path_dijkstra = dijkstra(edge, src, dst)
    for n in range(len(path_dijkstra[1]) - 1):
        for o in range(len(edge_port)):
            if (path_dijkstra[1][n] == edge_port[o][0][0] and path_dijkstra[1][n + 1] ==
                    edge_port[o][1][0]):
                di_dijkstra.append(edge_port[o][0][0])
                pi_dijkstra.append(edge_port[o][0][1])
                de_dijkstra.append(edge_port[o][1][0])
                pe_dijkstra.append(edge_port[o][1][1])
                if "of:" in edge_port[o][0][0]:
                    de.append(edge_port[o][0][0])
                    pe.append(edge_port[o][0][1])
                if "of:" in edge_port[o][1][0]:
                    di.append(edge_port[o][1][0])
                    pi.append(edge_port[o][1][1])
    return path_dijkstra, di_dijkstra, pi_dijkstra, de_dijkstra, pe_dijkstra, di, pi, de, pe

def install_best_dijkstra_path(src, dst, priority):  # Primary Path Intent Installation
    path_dijkstra, dij, pij, dej, pej, di, pi, de, pe = calculate_dijkstra_path(src, dst)
    path = " - ".join(path_dijkstra[1])
    print("Dijkstra's Shortest Path Calculation Result: \n")
    print(path)
    print("Total Cost for this path: {cost}\n".format(cost=path_dijkstra[0]))
    print("------------------------------------------------------------------------------------------")
    print("Installing Best-Path Dijkstra's Shortest Path..\n")
    intent = IntentManager()
    intent.del_intent()

    # Install Forwarding Intent
    print("Forwarding Intents based on Path Information")
    if len(di) == len(de):
        for n in range(len(de)):
            response = intent_p2p_install(pi[n], di[n], pe[n], de[n], priority)
            print("{no}.  Ingress = {di}:{pi} <--->  Egress = {de}:{pe} : status {stt}".format(no=n + 1, di=di[n],
                                                                                pi=pi[n], de=de[n],
                                                                                pe=pe[n],
                                                                                stt=response))
    # Install Reverse Forwarding Intent
    print("Reverse/Backwarding Intents based on Path Information")
    if len(di) == len(de):
        for n in range(len(de)):
            response = intent_p2p_install(pe[n], de[n], pi[n], di[n], priority)
            print("{no}.  Ingress = {de}:{pe} <--->  Egress = {di}:{pi} : status {stt}".format(no=n + 1, di=di[n],
                                                                                pi=pi[n],
                                                                                de=de[n], pe=pe[n],
                                                                                stt=response))

def generate_redundant_edge(src, dst):  # Generate redundant Topology
    edges, dev_in, dev_eg, edge_port = generate_edge()
    path_dijkstra, dij, pij, dej, pej, di, pi, de, pe = calculate_dijkstra_path(src, dst)
    delete_edge = []
    redundant_edges = []
    for y in range(len(edges)):
        for x in range(len(dij)):
            if edges[y][0] == dij[x] and edges[y][1] == dej[x]:
                if "of:" in edges[y][0] and "of:" in edges[y][1]:
                    delete_edge.append(edges[y])
            if edges[y][1] == dij[x] and edges[y][0] == dej[x]:
                if "of:" in edges[y][1] and "of:" in edges[y][0]:
                    delete_edge.append(edges[y])
    delete_edge.sort()
    even = [delete_edge[n] for n in range(len(delete_edge)) if n % 2 == 0]
    odd = [delete_edge[n] for n in range(len(delete_edge)) if n % 2 != 0]
    new_del_edge = list(zip(even, odd))

    for n in range(len(new_del_edge)):
        temp_edge = edges[:]
        redundant_edges.append(temp_edge)

    for n in range(len(redundant_edges)):
        redundant_edges[n].remove(new_del_edge[n][0])
        redundant_edges[n].remove(new_del_edge[n][1])

    for i in range(len(delete_edge)):
        edges.remove(delete_edge[i])
    return redundant_edges, edge_port

def calculate_redundant_dijkstra_path(src, dst):  # Calculate All Redundant Path
    redundant_edge, edge_port = generate_redundant_edge(src, dst)
    di_dijkstra = []
    de_dijkstra = []
    pi_dijkstra = []
    pe_dijkstra = []
    di = []
    de = []
    pi = []
    pe = []

    redundant_dijkstra = []
    redundant_di_dijkstra = []
    redundant_de_dijkstra = []
    redundant_pi_dijkstra = []
    redundant_pe_dijkstra = []
    redundant_di = []
    redundant_de = []
    redundant_pi = []
    redundant_pe = []
    new_redundant_dijkstra = []

    for edge in redundant_edge:
        path_dijkstra = dijkstra(edge, src, dst)
        redundant_dijkstra.append(path_dijkstra)

    redundant_dijkstra.sort()
    for i in redundant_dijkstra:
        if i not in new_redundant_dijkstra:
            new_redundant_dijkstra.append(i)

    for index in range(len(new_redundant_dijkstra)):
        for n in range(len(new_redundant_dijkstra[index][1]) - 1):
            for o in range(len(edge_port)):
                if (new_redundant_dijkstra[index][1][n] == edge_port[o][0][0] and
                        new_redundant_dijkstra[index][1][n + 1] == edge_port[o][1][0]):
                    di_dijkstra.append(edge_port[o][0][0])
                    pi_dijkstra.append(edge_port[o][0][1])
                    de_dijkstra.append(edge_port[o][1][0])
                    pe_dijkstra.append(edge_port[o][1][1])
                    if "of:" in edge_port[o][0][0]:
                        de.append(edge_port[o][0][0])
                        pe.append(edge_port[o][0][1])
                    if "of:" in edge_port[o][1][0]:
                        di.append(edge_port[o][1][0])
                        pi.append(edge_port[o][1][1])
        redundant_di_dijkstra.append(di_dijkstra)
        redundant_de_dijkstra.append(de_dijkstra)
        redundant_pi_dijkstra.append(pi_dijkstra)
        redundant_pe_dijkstra.append(pe_dijkstra)
        redundant_di.append(di)
        redundant_de.append(de)
        redundant_pi.append(pi)
        redundant_pe.append(pe)
    return (new_redundant_dijkstra, redundant_di_dijkstra, redundant_pi_dijkstra, redundant_de_dijkstra,
            redundant_pe_dijkstra, redundant_di, redundant_pi, redundant_de, redundant_pe)

def install_best_redundant_dijkstra_path(src, dst, priority):  # Best Redundant Path Intent Installation
    path_dijkstra, dij, pij, dej, pej, di, pi, de, pe = calculate_redundant_dijkstra_path(src, dst)
    print("Dijkstra's Shortest Path Calculation Result: \n")
    print("\nTotal Redundant Path: {total}\n".format(total=len(path_dijkstra)))
    for i in range(len(path_dijkstra)):
        path = " - ".join(path_dijkstra[i][1])
        print("{no}: {path}".format(no=i + 1, path=path))
        print("Total Cost for this path: {cost}\n".format(cost=path_dijkstra[i][0]))
    print("------------------------------------------------------------------------------------------")
    print("Installing Best-Path Redundant Dijkstra's Shortest Path...\n")

    # Install Forwarding Intent
    print("Forwarding Intents based on Path Information")
    if len(di[0]) == len(de[0]):
        for n in range(len(path_dijkstra[0][1]) - 2):
            response = intent_p2p_install(pi[0][n], di[0][n], pe[0][n], de[0][n], priority)
            print(
                "{no}.  Ingress = {di}:{pi} <--->  Egress = {de}:{pe} : status {stt}".format(no=n + 1, di=di[0][n],
                                                                                pi=pi[0][n],
                                                                                de=de[0][n],
                                                                                pe=pe[0][n],
                                                                                stt=response))
    # Install Reverse Forwarding Intent
    print("\n")
    print("Reverse/Backwarding Intents based on Path Information")
    if len(di[0]) == len(de[0]):
        for n in range(len(path_dijkstra[0][1]) - 2):
            response = intent_p2p_install(pe[0][n], de[0][n], pi[0][n], di[0][n], priority)
            print(
                "{no}.  Ingress = {de}:{pe} <--->  Egress = {di}:{pi} : status {stt}".format(no=n + 1, di=di[0][n],
                                                                                pi=pi[0][n],
                                                                                de=de[0][n],
                                                                                pe=pe[0][n],
                                                                                stt=response))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 15 13:03:19 2019

@author: thomas
"""

import numpy as np
import matplotlib.pyplot as plt
from scipy import special

x = np.arange(0,8,0.01)
font = {'family' : 'serif',
        'weight' : 'normal',
        'size'   : 16}
plt.rc('font', **font)

# Plot Q function
plt.figure(num=1,figsize=(10,8))
plt.xlabel('$x$',**font)
plt.ylabel('$Q(x)$')
plt.semilogy(x,0.5*special.erfc(x/np.sqrt(2.0)))
plt.legend(bbox_to_anchor=(1.3, 1.05))
plt.tight_layout()
plt.grid()
plt.savefig('Qfunction.png')


# https://leetcode.com/problems/01-matrix/discuss/101102/Short-solution-Each-path-needs-at-most-one-turn 顺时针旋转4次的方法
# 如何顺时针，逆时针90度旋转矩阵方法非常值得学习
# t = [[1, 2], [3, 4]]
# 顺时针旋转90度
# t2 = list(map(list, zip(*t[::-1])))
# 逆时针旋转90度
# t3 = list(map(list, zip(*t)))[::-1]

# 多次DP
from itertools import product
class Solution:
    def updateMatrix(self, matrix: List[List[int]]) -> List[List[int]]:
        m, n = len(matrix), len(matrix[0])
        ret = [[float('inf')] * n for _ in range(m)]
        # 第一次DP遍历，从左上往右下
        for i, j in product(range(m), range(n)):
            if matrix[i][j]:
                c = float('inf') if i == 0 else ret[i-1][j] + 1
                r = float('inf') if j == 0 else ret[i][j-1] + 1
                ret[i][j] = min(c, r)
            else:
                ret[i][j] = 0
        # 第二次DP遍历，从右下往左上
        for i, j in product(range(m-1, -1, -1), range(n-1, -1, -1)):
            if ret[i][j] > 0:
                c = float('inf') if i == m-1 else ret[i+1][j] + 1
                r = float('inf') if j == n-1 else ret[i][j+1] + 1
                ret[i][j] = min(ret[i][j], c, r)
        return ret

# from itertools import product
# from collections import deque
# # 多源BFS搜索
# class Solution:
#     def updateMatrix(self, matrix: List[List[int]]) -> List[List[int]]:
#         m, n = len(matrix), len(matrix[0])
#         check = [[True] * n for _ in range(m)]
#         dirs = [(-1, 0), (1, 0), (0, -1), (0, 1)]
#         dq = deque()
#         for i, j in product(range(m), range(n)):
#             if matrix[i][j] == 0:
#                 dq.append((i, j))
#                 check[i][j] = False
#         step = 0
#         while dq:
#             sz = len(dq)
#             for _ in range(sz):
#                 x, y = dq.popleft()
#                 matrix[x][y] = step
#                 for u, v in dirs:
#                     if 0 <= x+u < m and 0 <= y+v < n and check[x+u][y+v]:
#                         dq.append((x+u, y+v))
#                         check[x+u][y+v] = False
#             step += 1
#         return matrix

# # 普通BFS
# from itertools import product
# from collections import deque
# class Solution:
#     def updateMatrix(self, matrix: List[List[int]]) -> List[List[int]]:
#         m, n = len(matrix), len(matrix[0])
#         dirs = [(-1, 0), (1, 0), (0, -1), (0, 1)]
#         ret = [[-1] * n for _ in range(m)]
#         for i, j in product(range(m), range(n)):
#             if 0 == matrix[i][j]:
#                 ret[i][j] = 0

#         def bfs(i, j):
#             dq = deque([(i, j)])
#             count = 0
#             seen = {(i, j)}
#             while dq:
#                 sz = len(dq)
#                 for _ in range(sz):
#                     x, y = dq.popleft()
#                     if ret[x][y] == 0:
#                         return count
#                     for u, v in dirs:
#                         if 0 <= x+u < m and 0 <= y+v < n and (x+u, y+v) not in seen:
#                             dq.append((x+u, y+v))
#                             seen.add((x+u, y+v))
#                 count += 1

#         for i, j in product(range(m), range(n)):
#             if -1 == ret[i][j]:
#                 ret[i][j] = bfs(i, j)
#         return ret


import datetime
import os, sys
import pprint
from videotools.converter import *
import videotools.chunks as chunks
from videotools.transcribe import *
from videotools.talkgpt import *
from videotools.download_video import *
import os
import moviepy.editor as mp
from pydub import AudioSegment
from pydub.utils import make_chunks
import argparse
from multiprocessing import Pool

def main(args_videotools):

    parser = argparse.ArgumentParser(description='VideoTools Arguments')
    print("\n\n"+get_completion("Repita 'Eu estou funcionando e sou o GPT' : ")+"\n\n")

    #convertendo de mp4 para mp3
    # videoMp4=args[1]
    parser.add_argument('-videoMp4','-v', type=str, help='video path')

    # destino=args[2]
    parser.add_argument('-destino','-d', type=str, help='destination path')
    parser.add_argument('-leng','-lg', type=str, help='language')
    parser.add_argument('-timechunk','-tc', type=int, help='timechunk')
    parser.add_argument('-assuntos','-as', type=int, help='Number of subjects')
    parser.add_argument('-url','-u', type=str, help='url')

    args_videotools = parser.parse_args()

    print(args_videotools.videoMp4)

    videoMp4=args_videotools.videoMp4
    destino=args_videotools.destino

    #baixa video do youtube se tiver url
    if args_videotools.url:
        #filename video_name_date
        date_now = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        video_name = "video_"+str(date_now)
        print(args_videotools.url)
        print(video_name)
        dowload_youtube(args_videotools.url,video_name,destino)
        #print("baixou!")
        videoMp4=os.path.join(destino,video_name+".mp4")
    else:
        video_name = os.path.splitext(videoMp4)[0]
        video_name = video_name.split("/")[-1]

    try:
        # leng=args[3]
        leng=args_videotools.leng if args_videotools.leng else "pt-BR"
    except:
        leng="pt-BR"
        #ingles: en-US
        #portugues: pt-BR
        #espanhol: es-ES
        #frances: fr-FR
        #italiano: it-IT

    try:
        timechunk= args_videotools.timechunk if args_videotools.timechunk else 30000
        print(str(timechunk) + "timechunk")
    except:
        timechunk=30000

    try:
        subjects= args_videotools.assuntos if args_videotools.assuntos else 3
    except:
        subjects=3

    #video path example: windows:  "C:\\Users\\User\\Desktop\\video.mp4" linux: "/home/user/video.mp4"

    converter(videoMp4,"mp4","mp3", os.path.relpath(destino))

    ARQUIVO_MP3 = os.path.join(destino,video_name+".mp3")
    leng_no_=leng.split("-")[0]
    transcribe_py=os.path.realpath("videotools/transcribe.py")
    os.system(f"python3 {transcribe_py} -mp3 {ARQUIVO_MP3} -lang {leng_no_} > {os.path.join(destino,'transcricao.txt')}")

    print("transcrição completa!! -- "+os.path.join(destino,'transcricao.txt'))

    with open(os.path.join(destino,"transcricao.txt"),"r") as f:
        texto=f.read()
        texto.split("INICIANDO TRANSRIÇÃO:")[1].split("TERMINANDO TRANSCRIÇÃO---")[0]

    #gerando assuntos
    we_got_it=False

    a_geracao=gerar_assuntos(texto,destino,subjects,leng)
    while(not we_got_it):
        print("\n\n\nTENTANDO IDENTIFICAR CORTES!! ")
        texto_for_gpt=f""" SUMARIZE ESSES ASSUNTOS SEJA EXATO E COM POUCAS PALAVRAS E DIVIDA EM {subjects} ASSUNTOS retornando EXATAMENTE [t_inicial | t_final ] -  [Assunto]

                                {a_geracao}

                LEMBRE-SE DE RETORNAR EXATAMENTE NO PADRÃO [t_inicial | t_final ] -  [Assunto] E EM {leng}!
                POIS EU VOU PEGAR OS CORTES USANDO ESSA EXPRESSÃO: "t_inicial":linha.split("|")[0].split("[")[1].split("]")[0].strip(),"t_final":linha.split("|")[1].split("]")[0].strip(),"assunto":linha.split("|")[1].split("]")[1].strip()
                """
        with open(os.path.join(destino,"last_send.txt"),"w") as f:
            f.write(texto_for_gpt)
        resposta =get_completion(texto_for_gpt)
        print("Resposta:\n\n"+resposta+"\n\n")

        with open(os.path.join(destino,"cortes.txt"),"w") as f:
            f.write(resposta)

        #obtendo cortes
        # [00:07.660 | 01:10.440] - Elogio e comentário sobre o livro de Vardin Rogovim e evento em Belo Horizonte.
        # [01:10.440 | 03:09.500] - Discussão sobre a história do movimento comunista no Brasil e a influência do Stalinismo.
        # [03:09.500 | 05:08.040] - Exploração do impacto histórico e político do Stalinismo na União Soviética, ressaltando a obra de Vadim Rogovim.
        # [05:08.040 | 06:58.020] - Análise crítica do surgimento e efeitos do Stalinismo na história soviética e na tradição marxista.
        # [06:58.020 | 16:53.540] - Detalhes sobre o conteúdo e a importância da obra de Rogovim, além de convite para o evento em Belo Horizonte e considerações finais.
        cortes=[]
        clip=[]
        with open(os.path.join(destino,"cortes_resumo.txt"),"w") as fr:
            try:
                with open(os.path.join(destino,"cortes.txt"),"r") as f:
                    linhas=f.readlines()
                    for linha in linhas:
                        try:
                            clip={"t_inicial":linha.split("|")[0].split("[")[1].split("]")[0].strip(),"t_final":linha.split("|")[1].split("]")[0].strip(),"assunto":linha.split("|")[1].split("]")[1].strip()}
                        except:
                            pass
                        cortes.append(clip)
                        fr.write(f"""{clip['t_inicial']} - {clip['assunto']} \n""")
                    print("CONSEGUIU OS CORTES:")
                    print(cortes)

                #cortando videos e gerando clips na pasta [destino]/clips
                print("CORTANDO VIDEOS E GERANDO CLIPS NA PASTA [destino]/clips")
                video = mp.VideoFileClip(videoMp4)
                if not os.path.exists(os.path.join(destino,"clips")):
                    os.makedirs(os.path.join(destino,"clips"))
                for corte in cortes:
                    try:

                        if(corte["t_inicial"]!="t_inicial" and corte["t_final"]!="t_final"):

                            get_Segundos = lambda x: float(x.split(":")[0])*60+float(x.split(":")[1].split(".")[0])+float(x.split(":")[1].split(".")[1])/1000
                            t_inicial=corte["t_inicial"]
                            t_final=corte["t_final"]
                            #coloca os tempos no padrão 00:00:00.00
                            if(len(t_inicial.split(":"))==2):
                                t_inicial="00:"+t_inicial

                            print(f"Cortando de {t_inicial} até {t_final} -  clip: {corte['assunto']}")

                            clip = video.subclip(t_inicial,t_final)
                            #nome do arquivo encodeado para tirar caracteres especiais
                            nome_arquivo=corte["assunto"].encode('ascii', 'ignore').decode('ascii')
                            #tira todos os caracteres especiais
                            nome_arquivo = ''.join(e for e in nome_arquivo if e.isalnum() or e==" ")

                            nome_clip=os.path.join(destino,"clips",nome_arquivo.replace(" ","_")+".mp4")

                            #utilizando multiprocessing para cortar os videos
                            clip.write_videofile(nome_clip,codec="libx264",threads=4)

                    except Exception as e:
                        print("\n\nerro ao cortar o video : "+corte["assunto"]+"\n\n")
                        print(e)
                        continue
                print("CORTES COMPLETOS!!")
                we_got_it=True
            except Exception as e:
                print("\n\nerro ao obter os cortes, tente novamente: "+str(e))
                print("Tentando Novamente!!")
                we_got_it=False

    ...

def gerar_assuntos(transcricao:str,destino:str,subjects:int,leng:str,max_length:int=200):
    #obtendo cortes

    #contando linhas da transcrição
    linhas = transcricao.split("\n")
    respostaFinal=""

    for i in range(0,len(linhas),max_length):
        #obtendo a parte
        parte = linhas[i:i+max_length]
        #escrevendo em um arquivo
        with open(os.path.join(destino,"transcricao.txt"),"w") as f:
            f.write("\n".join(parte))
        #gerando assuntos

        with open(os.path.join(destino,"transcricao.txt"),"r") as f:
            texto=f.read()

            resposta=""

            while(resposta==""):

                resposta =get_completion(f"""
                Leia a transcrição abaixo e SUMARIZE EM POUCAS PALAVRAS ({subjects} assuntos) retornando EXATAMENTE [t_inicial | t_final ] -  [Assunto]

                transcrição:
                                {texto}

                LEMBRE-SE DE RETORNAR EXATAMENTE NO PADRÃO [t_inicial | t_final ] -  [Assunto] E EM {leng}!
                POIS EU VOU PEGAR OS CORTES USANDO ESSA EXPRESSÃO: "t_inicial":linha.split("|")[0].split("[")[1].split("]")[0].strip(),"t_final":linha.split("|")[1].split("]")[0].strip(),"assunto":linha.split("|")[1].split("]")[1].strip()
                """)

            respostaFinal+=resposta+"\n"

            #tirar linhas em branco

    with open(os.path.join(destino,"cortes.txt"),"w") as f:
        f.write(respostaFinal)
    return respostaFinal

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv))


# Example taken from the LinkedIn Learning course named "Python Essential Training" by Bill Weinman. Link: https://www.linkedin.com/learning/python-essential-training-2/generators

def main():
    for i in inclusive_range(25):
        print(i, end=' ')
    print()

def inclusive_range(*args):
    """A custom version of the built-in 'range' generator function, one which also includes the end value."""
    numargs = len(args)
    start = 0
    step = 1

    if numargs == 0:
        raise TypeError(f'Expected at least 1 argument, got {numargs}.')
    elif numargs == 1:
        stop = args[0]
    elif numargs == 2:
        (start, stop) = args
    elif numargs == 3:
        (start, stop, step) = args
    else:
        raise TypeError(f'Expected at most 3 arguments, got {numargs}.')

    # Generator:
    i = start
    while i <= stop:
        yield i
        i += step

if __name__ == '__main__':
    main()


import argparse
import os
from exp.exp_pred import Exp_pred
from exp.exp_mae import Exp_mae

from data.stock_data_handle import Stock_Data
import utils.tools as utils
import time

import pdb
import random
import torch
import numpy as np

fix_seed = 2022
random.seed(fix_seed)
torch.manual_seed(fix_seed)
np.random.seed(fix_seed)

parser = argparse.ArgumentParser(description='[Transformer] Long Sequences Forecasting')

parser.add_argument('--model', type=str, default='Transformer',help='model of the experiment')
parser.add_argument('--project_name', type=str, default='baseline',help='name of the experiment')

parser.add_argument('--data_name', type=str, default='CSI', help='')
parser.add_argument('--data_type', type=str, default='stock', help='stock')
parser.add_argument('--root_path', type=str, default='data/', help='root path of the data file')
parser.add_argument('--full_stock_path', type=str, default='processed_data/CSI/', help='root path of the data file')

parser.add_argument('--exp_type', type=str, default='pred', help='[mae|pred]')

parser.add_argument('--seq_len', type=int, default=60, help='input series length')
parser.add_argument('--label_len', type=int, default=1, help='help series length')
parser.add_argument('--pred_len', type=int, default=1, help='predict series length')

parser.add_argument('--enc_in', type=int, default=96, help='encoder input size: cov+technical indicators')
parser.add_argument('--dec_in', type=int, default=96, help='decoder input size')
parser.add_argument('--c_out', type=int, default=96, help='output size')

parser.add_argument('--short_term_len', type=int, default=1, help='short term prediction len')
parser.add_argument('--long_term_len', type=int, default=5, help='long term prediction len')
parser.add_argument('--pred_type', type=str, default='long_term_len', help='type of prediction')

parser.add_argument('--d_model', type=int, default=128, help='dimension of model')
parser.add_argument('--n_heads', type=int, default=4, help='num of heads')
parser.add_argument('--e_layers', type=int, default=2, help='num of encoder layers')
parser.add_argument('--d_layers', type=int, default=1, help='num of decoder layers')
parser.add_argument('--d_ff', type=int, default=256, help='dimension of fcn')

parser.add_argument('--dropout', type=float, default=0.05, help='dropout')

parser.add_argument('--activation', type=str, default='gelu',help='activation')
parser.add_argument('--num_workers', type=int, default=10, help='data loader num workers')

parser.add_argument('--rank_alpha', type=float, default=0.1, help='weight of rank loss') # adjust

parser.add_argument('--itr', type=int, default=2, help='each params run iteration')
parser.add_argument('--train_epochs', type=int, default=30, help='train epochs')
parser.add_argument('--batch_size', type=int, default=32, help='input data batch size')
parser.add_argument('--patience', type=int, default=3, help='early stopping patience')
parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate')
parser.add_argument('--adjust_interval', type=int, default=1, help='lr adjust interval')
parser.add_argument('--des', type=str, default='pred',help='exp description')
parser.add_argument('--loss', type=str, default='mse',help='loss function')
parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate')

# GPU
parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu')
parser.add_argument('--gpu', type=int, default=0, help='gpu')
parser.add_argument('--use_multi_gpu', action='store_true', help='use multiple gpus', default=False)
parser.add_argument('--devices', type=str, default='0,1,2,3', help='device ids of multile gpus')

args = parser.parse_args()
args.use_gpu = True if torch.cuda.is_available() and args.use_gpu else False

if args.use_gpu and args.use_multi_gpu:
    args.dvices = args.devices.replace(' ', '')
    device_ids = args.devices.split(',')
    args.device_ids = [int(id_) for id_ in device_ids]
    args.gpu = args.device_ids[0]

print(args)

exp_dict = {'pred': Exp_pred, 'mae': Exp_mae}
data_type_dict = {'stock': Stock_Data}
Exp = exp_dict[args.exp_type]
data =  data_type_dict[args.data_type](
        root_path=args.root_path,
        dataset_name=args.data_name,
        full_stock_path=args.full_stock_path,
        size=[args.seq_len, args.label_len, args.pred_len],
        prediction_len=[args.short_term_len, args.long_term_len]
        )

# pdb.set_trace()

for ii in range(args.itr):
    id = utils.generate_id()
    setting = '{}_{}_{}_alpha{}_sl{}_pl{}_enc{}_cout{}_dm{}_nh{}_el{}_dl{}_df{}_{}_{}_dt{}_id{}'.format(args.exp_type, args.project_name, args.data_name, str(args.rank_alpha).replace('.','_'),
                args.seq_len, args.pred_len, args.enc_in, args.c_out,
                args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.des, ii, args.data_name, id)

    exp = Exp(args, data, id)
    print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting))
    print('Task id: ',id)
    start = time.time()
    exp.train(setting)
    end = time.time()
    print("Training Time:",end-start)

    print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting))
    exp.test(setting)


# this programe showcase boolean operation
'''Python boolean operators are spelled out as the words "and" "or" "not",
 instead of the && syntax in other languages.'''

def a_bigger(a, b):
    if a > b and (a-b) >= 2:
        return True
    else:
        return False

a_bigger(0, 2)


import collections

from descriptor_bulkfood2 import Validated, Quantity, NonBlank

class EntityMeta(type):
    """Metaclass for business entities with validated fields"""

    def __init__(cls, name, bases, attr_dict):
        # call __init__ on the superclass
        super().__init__(name, bases, attr_dict)
        for key, attr in attr_dict.items():
            if isinstance(attr, Validated):
                type_name = type(attr).__name__
                attr.storage_name = '_{}#{}'.format(type_name, key)

class Entity(metaclass=EntityMeta):
    """Business entity with validated fields

    This class exists for convenience only: the user can just subclass
    Entity and not worry about EntityMeta."""

class EntityMeta2(type):
    """Metaclass for business entities with validated fields"""

    @classmethod
    def __prepare__(mcs, name, bases):
        """
        __prepare__ is invoked before __new__ in the metaclass to create the mapping
         that will be filled with the attrs from the class body.
        :param mcs: metaclass
        :param name: name of the class to be constructed
        :param bases: tuple of base classes of the class to be constructed
        :return: a mapping that will be receieved as the last arg by __new__ and then
            __init__ when the metaclass builds a new class
        """
        # return an empty OrderedDict instance, where the class attrs will be stored
        return collections.OrderedDict()

    def __init__(cls, name, bases, attr_dict):
        super().__init__(name, bases, attr_dict)
        # create a _field_names attribute in the class under construction
        cls._field_names = []
        # attr_dict is the OrderDict obtained by calling __prepare__
        for key, attr in attr_dict.items():
            # add the name of each Validated field found to _field_names
            if isinstance(attr, Validated):
                type_name = type(attr).__name__
                attr.storage_name = '_{}#{}'.format(type_name, key)
                cls._field_names.append(key)

class Entity2(metaclass=EntityMeta2):
    """Business entity with validated fields"""

    @classmethod
    def field_names(cls):
        for name in cls._field_names:
            yield name

class LineItem(Entity2):
    """
    >>> for name in LineItem.field_names():
    ...     print(name)
    ...
    description
    weight
    price
    """
    description = NonBlank()
    weight = Quantity()
    price = Quantity()

    def __init__(self, description, weight, price):
        self.description = description
        self.price = price
        self.weight = weight

    def subtotal(self):
        return self.weight * self.price


from machine import Pin, SoftI2C
import ssd1306
from time import sleep
from random import randint
from math import sin, cos, radians

class Graph:
    def __init__(self, oled):
        self.oled = oled
        self.step_noise_x = 2
        self.step_noise_y = 2

    def draw_graph(self, oled):
        for y in range(oled.height):
            pirnt(y)

    def draw_sin(self, start=0, stop=7200, step=7):
        x_s = 0
        for x in range(start, stop, step):
            y = self.oled.height/2 + self.oled.height/2 * sin(radians(x))
            if x_s > self.oled.width:
                x_s = 0
                self.oled.fill(0)
            self.oled.pixel(x_s, int(y), 1)
            self.oled.show()
            x_s += 1

    def make_noise(self):
        while True:
            self.oled.fill(0)
            for y in range(0, self.oled.height, self.step_noise_y):
                for x in range(0, self.oled.width, self.step_noise_x):
                    self.oled.pixel(x + randint(0, 5), y + randint(0, 5), 1)
            self.oled.show()

def main():
    i2c = SoftI2C(scl=Pin(25), sda=Pin(26))

    oled_width = 128
    oled_height = 64
    oled = ssd1306.SSD1306_I2C(oled_width, oled_height, i2c)

    gr = Graph(oled)
    gr.make_noise()

if __name__ == '__main__':
    main()


from cProfile import label
from operator import index
from re import I
from tkinter import Label
from typing import Counter
import cv2
from sys import argv
import numpy as np
import matplotlib.pyplot as plt

mode = argv[1]

if mode == 'news':
	dir = 'news_out_modified/'
	amount = 1380
elif mode == 'ngc':
	dir = 'ngc_out_modified/'
	amount = 1060
elif mode == 'ftfm':
	dir = 'ftfm_out_modified/'
	amount = 770
else:
	print("Error input!")
	quit()

####################### Algorithm #######################

def evaluate(similarity1,similarity2,similarity3,similarity4,similarity5,weightedsim):
    # Evaluating the performance
    plt.figure("similarity - frame")
    plt.plot(similarity1, color='purple', label='Gray Hist')
    plt.plot(similarity2, color='yellow', label='RGB Hist')
    plt.plot(similarity3, color='red',  label='aHash')
    plt.plot(similarity4, color='green',  label='dHash')
    plt.plot(similarity5, color='blue',  label='pHash')
    plt.plot(weightedsim, color='aqua',  label='Weighted algorithm')
    plt.plot(thres,color='black', label='Threshold')
    plt.xlabel("frame")
    plt.ylabel("similarity")
    plt.legend(loc='best')
    plt.show()
    return

def create_gray_hist(image):
    #img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    hist = cv2.calcHist([image], [0], None, [256],  [0, 256])
    return hist

def hist_compare(image1, image2):

    hist1 = create_gray_hist(image1)
    hist2 = create_gray_hist(image2)
    # 進行三種方式的直方圖比較
    match1 = cv2.compareHist(hist1, hist2, cv2.HISTCMP_BHATTACHARYYA)
    match2 = cv2.compareHist(hist1, hist2, cv2.HISTCMP_CORREL)
    similarity = ((1-match1) + match2)/2
    return similarity

def classify_hist_with_split(image1, image2, size=(256, 256)):
    # RGB每個通道的直方圖相似度
    # 將圖像resize後，分離爲RGB三個通道，再計算每個通道的相似值
    image1 = cv2.resize(image1, size)
    image2 = cv2.resize(image2, size)
    sub_image1 = cv2.split(image1)
    sub_image2 = cv2.split(image2)
    sub_data = 0
    for im1, im2 in zip(sub_image1, sub_image2):
        sub_data += hist_compare(im1, im2)
    sub_data = sub_data / 3
    return sub_data

def aHash(img):
    # 均值哈希算法
    # 縮放爲8*8
    img = cv2.resize(img, (8, 8))
    # 轉換爲灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # s爲像素和初值爲0，hash_str爲hash值初值爲''
    s = 0
    hash_str = ''
    # 遍歷累加求像素和
    for i in range(8):
        for j in range(8):
            s = s+gray[i, j]
    # 求平均灰度
    avg = s/64
    # 灰度大於平均值爲1相反爲0生成圖片的hash值
    for i in range(8):
        for j in range(8):
            if gray[i, j] > avg:
                hash_str = hash_str+'1'
            else:
                hash_str = hash_str+'0'
    return hash_str

def dHash(img):
    # 差值哈希算法
    # 縮放8*8
    img = cv2.resize(img, (9, 8))
    # 轉換灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    hash_str = ''
    # 每行前一個像素大於後一個像素爲1，相反爲0，生成哈希
    for i in range(8):
        for j in range(8):
            if gray[i, j] > gray[i, j+1]:
                hash_str = hash_str+'1'
            else:
                hash_str = hash_str+'0'
    return hash_str

def pHash(img):
    # 感知哈希算法
    # 縮放32*32
    img = cv2.resize(img, (32, 32))   # , interpolation=cv2.INTER_CUBIC

    # 轉換爲灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 將灰度圖轉爲浮點型，再進行dct變換
    dct = cv2.dct(np.float32(gray))
    # opencv實現的掩碼操作
    dct_roi = dct[0:8, 0:8]

    hash = []
    average = np.mean(dct_roi)
    for i in range(dct_roi.shape[0]):
        for j in range(dct_roi.shape[1]):
            if dct_roi[i, j] > average:
                hash.append(1)
            else:
                hash.append(0)
    return hash

def cmpHash(hash1, hash2):
    # Hash值對比
    # 算法中1和0順序組合起來的即是圖片的指紋hash。順序不固定，但是比較的時候必須是相同的順序。
    # 對比兩幅圖的指紋，計算漢明距離，即兩個64位的hash值有多少是不一樣的，不同的位數越小，圖片越相似
    # 漢明距離：一組二進制數據變成另一組數據所需要的步驟，可以衡量兩圖的差異，漢明距離越小，則相似度越高。漢明距離爲0，即兩張圖片完全一樣
    n = 0
    # hash長度不同則返回-1代表傳參出錯
    if len(hash1) != len(hash2):
        return -1
    # 遍歷判斷
    for i in range(len(hash1)):
        # 不相等則n計數+1，n最終爲相似度
        if hash1[i] != hash2[i]:
            n = n + 1
    return n

####################### Performance Analysis #######################

def pfanaly(similarity):
    correcthit = 0
    falsepositive = 0
    hitprebound2 = 0
    hitprebound3 = 0

    for i in range(1,amount):
        hit=0
        if similarity[i,0] < threshold:

            if mode == 'news':
                f1 = open('news_ground.txt')
                for line in f1:
                    if(i == int(line)):
                        hit=1
                        correcthit += 1
                        break

                if (hit == 0):
                    falsepositive += 1

                f1.close()

            elif mode == 'ngc':
                f2 = open('ngc_ground.txt')
                for line in f2:
                    if '~' in line:
                        prebound2=line.split("~")[0]
                        postbound2=line.split("~")[1]

                        if( (i <= int(postbound2)) & (i >= int(prebound2)) ):
                            hit=1
                            if(hitprebound2 != int(prebound2)):
                                hitprebound2 = int(prebound2)
                                correcthit += 1
                                break
                    else:
                        if(i == int(line)):
                            hit=1
                            correcthit += 1
                            break

                if (hit == 0):
                    falsepositive += 1

                f2.close()

            elif mode == 'ftfm':
                f3 = open('ftfm_ground.txt')
                for line in f3:
                    if '~' in line:
                        prebound3=line.split("~")[0]
                        postbound3=line.split("~")[1]
                        if( (i <= int(postbound3)) & (i >= int(prebound3)) ):
                            hit=1
                            if(hitprebound3 != int(prebound3)):
                                hitprebound3 = int(prebound3)
                                correcthit += 1
                                break
                    else:
                        if(i == int(line)):
                            hit=1
                            correcthit += 1
                            break

                if (hit == 0):
                    falsepositive += 1

                f3.close()

    print("Correct hit amount is : ",correcthit)
    print("False positive amount is : ",falsepositive)

    if mode == 'news':
        recall = correcthit/7 #groundhit1 = 7
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision
    elif mode == 'ngc':
        recall = correcthit/36 #groundhit2 = 36
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision
    elif mode == 'ftfm':
        recall = correcthit/27 #groundhit3 = 27
        if(correcthit + falsepositive) == 0:
            precision = 0
        else:
            precision = correcthit/(correcthit + falsepositive)
        return recall, precision

#######################Main#######################

recall1 = np.zeros(25)
precision1 = np.zeros(25)
recall2 = np.zeros(25)
precision2 = np.zeros(25)
recall3 = np.zeros(25)
precision3 = np.zeros(25)
recall4 = np.zeros(25)
precision4 = np.zeros(25)
recall5 = np.zeros(25)
precision5 = np.zeros(25)
recall = np.zeros(25)
precision = np.zeros(25)
index = 0

for m in range (0,25):
    threshold = 0.5 + m*0.02

    similarity1 = np.ones((amount,1))
    similarity2 = np.ones((amount,1))
    similarity3 = np.ones((amount,1))
    similarity4 = np.ones((amount,1))
    similarity5 = np.ones((amount,1))
    weightedsim = np.ones((amount,1))
    thres = np.full((amount,1),threshold)

    for i in range(1,amount):
        j=str(i-1)
        #print(j)

        k=str(i)
        #print(k)

        if mode == 'ngc':
            testpath = dir + j + '.jpeg'
            #print (testpath)
            test_img = cv2.imread(testpath)
            test_img = cv2.resize(test_img, (128,128), interpolation = cv2.INTER_AREA)

            datapath = dir + k + '.jpeg'
            #print (datapath)
            data_img = cv2.imread(datapath)
            data_img = cv2.resize(data_img, (128,128), interpolation = cv2.INTER_AREA)
        else:
            testpath = dir + j + '.jpg'
            #print (testpath)
            test_img = cv2.imread(testpath)
            test_img = cv2.resize(test_img, (128,128), interpolation = cv2.INTER_AREA)

            datapath = dir + k + '.jpg'
            #print (datapath)
            data_img = cv2.imread(datapath)
            data_img = cv2.resize(data_img, (128,128), interpolation = cv2.INTER_AREA)

        score1 = hist_compare(test_img,data_img)
        similarity1[i,0] = score1

        score2 = classify_hist_with_split(test_img,data_img)
        similarity2[i,0] = score2

        hash1 = aHash(data_img)
        hash2 = aHash(test_img)
        score3 = cmpHash(hash1, hash2)
        score3 = (1-float(score3/64))
        similarity3[i,0] = score3

        hash1 = dHash(data_img)
        hash2 = dHash(test_img)
        score4 = cmpHash(hash1, hash2)
        score4 = (1-float(score4/64))
        similarity4[i,0] = score4

        hash1 = pHash(data_img)
        hash2 = pHash(test_img)
        score5 = cmpHash(hash1, hash2)
        score5 = (1-float(score5/64))
        similarity5[i,0] = score5

        weightedsim[i,0] = (score1+score2+score3+score4+score5)/5

    print("Gray hist comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity1[i,0] < threshold:
            print (i)

    print("\nRGB hist comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity2[i,0] < threshold:
            print (i)

    print("\naHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity3[i,0] < threshold:
            print (i)

    print("\ndHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity4[i,0] < threshold:
            print (i)

    print("\npHash comparison detects shot change at : ")
    for i in range (1,amount):
        if similarity5[i,0] < threshold:
            print (i)

    print("\nWeighted algorithms detects shot change at : ")
    for i in range (1,amount):
        if weightedsim[i,0] < threshold:
            print (i)

    result1=pfanaly(similarity1)
    recall1[index] = result1[0]
    precision1[index] = result1[1]

    result2=pfanaly(similarity2)
    recall2[index] = result2[0]
    precision2[index]= result2[1]

    result3=pfanaly(similarity3)
    recall3[index] = result3[0]
    precision3[index]= result3[1]

    result4=pfanaly(similarity4)
    recall4[index] = result4[0]
    precision4[index] = result4[1]

    result5=pfanaly(similarity5)
    recall5[index] = result5[0]
    precision5[index] = result5[1]

    result=pfanaly(weightedsim)
    recall[index] = result[0]
    #print(recall[index])
    precision[index] = result[1]
    #print(precision[index])

    index += 1

#print("Precision for Gray hist comparison is : ",precision1)
#print("Recall for Gray hist comparison is : ",recall1)

#print("Precision for RGB hist comparison is : ",precision2)
#print("Recall for RGB hist comparison is : ",recall2)

#print("Precision for aHash comparison is : ",precision3)
#print("Recall for aHash comparison is : ",recall3)

#print("Precision for dHash comparison is : ",precision4)
#print("Recall for dHash comparison is : ",recall4)

#print("Precision for pHash comparison is : ",precision5)
#print("Recall for pHash comparison is : ",recall5)

#print("Precision for weighted algorithms is : ",precision)
#print("Recall for weighted algorithms is : ",recall)

plt.figure("PR Curve")
plt.plot(recall1,precision1,label='Gray hist')
plt.plot(recall2,precision2,label='RGB hist')
plt.plot(recall3,precision3,label='aHash')
plt.plot(recall4,precision4,label='dHash')
plt.plot(recall5,precision5,label='pHash')
plt.plot(recall,precision,label='weighted algorithms')
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend(loc='best')
plt.show()

#hist = evaluate(similarity1,similarity2,similarity3,similarity4,similarity5,weightedsim)


"""
This is an experimental module.
Everything in this module is untested and probably incorrect.
Don't use them.

This is intended to be a place to develop new functions instead of
having an entirely new branch
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from numba import jit, njit
import os
import inspect
from intake import Schema

class DataSource:
    """
    A dummy datasource class replacing the intake datasource
    class to enable fast loading of this module
    This is a temporary ugly fix
    Intake catalog classes must be moved to a separate module
    """

    def __init__(self):
        pass

@jit
def v_cusum(array):
    """
    Calcuate cusum - numba version
    array
        numpy array
    returns
        pos and neg arrays
    """
    len(array)
    pos = [0]
    neg = [0]
    pos_val = 0
    neg_val = 0
    d = np.diff(array)[1:]
    for i in d:
        if i >= 0:
            pos_val += i
        else:
            neg_val += i
        pos.append(pos_val)
        neg.append(neg_val)
    return (pos, neg)

@jit
def sign_change(array):
    """
    Calcuate the sign change in an array
    If the current value is positive and previous value negative, mark as 1.
    If the current value is negative and previous value positive, mark as -1.
    In case of no change in sign, mark as 0
    """
    L = len(array)
    arr = np.empty(L)
    arr[0] = 0
    for i in range(1, L):
        # TO DO: Condition not handling edge case
        if (array[i] >= 0) & (array[i - 1] < 0):
            arr[i] = 1
        elif (array[i] <= 0) & (array[i - 1] > 0):
            arr[i] = -1
        else:
            arr[i] = 0
    return arr

def cusum(array):
    """
    Calcuate cusum
    array
        a pandas series with a timestamp or datetime index
    The cusum is just an aggregate of positive and negative differences
    returns
        pandas dataframe with positive and negative cumulatives,
        ratio, differences, regime change along with the original index
    """
    pos = [0]
    neg = [0]
    pos_val = 0
    neg_val = 0
    d = array.diff()[1:]
    for i in d:
        if i >= 0:
            pos_val += i
        else:
            neg_val += i
        pos.append(pos_val)
        neg.append(neg_val)
    df = pd.DataFrame({"pos": pos, "neg": neg}, index=array.index)
    df["neg"] = df["neg"].abs()
    df["d"] = df["pos"] - df["neg"]
    df["reg"] = sign_change(df.d.values)
    df["ratio"] = df["pos"] / df["neg"]
    return df

def percentage_bar(data, step):
    """
    Generate the number of timesteps taken for each
    equivalent step in price
    data
        numpy 1d array
    step
        step size
    """
    start = data[0]
    nextStep = start + step
    counter = 0
    steps = [start]
    period = [0]
    for d in data:
        if step >= 0:
            if d > nextStep:
                steps.append(nextStep)
                period.append(counter)
                nextStep += step
                counter = 0
            else:
                counter += 1
        elif step < 0:
            if d < nextStep:
                steps.append(nextStep)
                period.append(counter)
                nextStep += step
                counter = 0
            else:
                counter += 1

    # Final loop exit
    steps.append(nextStep)
    period.append(counter)
    return (steps, period)

def high_breach(s):
    """
    Given a series of values, returns a series
    with consecutive highs as values and timestamp as index
    s
        series with timestamp as index
    """
    highs = []
    ts = []
    max_val = 0
    index = s.index.values
    for i, v in enumerate(s.values):
        if v > max_val:
            highs.append(v)
            ts.append(index[i])
            max_val = v
    return pd.Series(highs, index=ts)

def low_breach(s):
    """
    Given a series of values, returns a series
    with consecutive lows as values and timestamp as index
    s
        series with timestamp as index
    """
    lows = []
    ts = []
    min_val = 1e9  # Just setting an extreme value
    index = s.index.values
    for i, v in enumerate(s.values):
        if v < min_val:
            lows.append(v)
            ts.append(index[i])
            min_val = v
    return pd.Series(lows, index=ts)

class ExcelSource(DataSource):
    container = "dataframe"
    name = "excel_loader"
    version = "0.0.1"
    partition_access = True

    def __init__(self, datapath, metadata=None):
        """
        Initialize with datapath and metadata
        datapath
            filename with entire path
        """
        self.filename = datapath
        self._source = pd.ExcelFile(self.filename)
        super(ExcelSource, self).__init__(metadata=metadata)

    def _get_schema(self):
        sheets = self._source.sheet_names
        return Schema(
            datashape=None,
            dtype=None,
            shape=None,
            npartitions=len(sheets),
            extra_metadata={"sheets": sheets},
        )

    def read_partition(self, sheet, **kwargs):
        """
        Read a specific sheet from the list of sheets
        sheet
            sheet to read
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        if sheet in self.metadata.get("sheets", []):
            return self._source.parse(sheet, **kwargs)
        else:
            return "No such sheet in the Excel File"

    def read(self, **kwargs):
        """
        Read all sheets into a single dataframe.
        Sheetname is added as a column
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        sheets = self.metadata.get("sheets")
        collect = []
        if len(sheets) > 1:
            for sheet in sheets:
                temp = self.read_partition(sheet, **kwargs)
                temp["sheetname"] = sheet
                collect.append(temp)
        return pd.concat(collect, sort=False)

    def _close(self):
        self._source.close()

class HDFSource(DataSource):
    """
    A simple intake container to load data from
    HDF5 fixed formats
    """

    container = "dataframe"
    name = "HDF5_fixed_loader"
    version = "0.0.1"
    partition_access = True

    def __init__(self, datapath, metadata=None, extension="h5"):
        """
        Initialize with datapath and metadata
        datapath
            filename or directory
            If the filename ends in any of the extensions given in the
            extension argument, then it is treated as a HDF5 file
        """
        self.source = datapath
        self._ext = extension
        # Check whether the given path is a directory or file
        if os.path.exists(datapath):
            if os.path.isfile(datapath):
                self._source_type = "file"
            else:
                self._source_type = "directory"
        else:
            print("Not a valid file or directory")
            return
        self._get_schema()
        super(HDFSource, self).__init__(metadata=metadata)

    def _get_schema(self):
        metadata = {"ext": self._ext, "src": self.source, "type": self._source_type}
        file_dict = {}
        if self._source_type == "directory":
            for root, directory, files in os.walk(self.source):
                for file in files:
                    filename = os.path.join(root, file)
                    if filename.endswith(self._ext):
                        file_dict[file] = filename
            metadata.update({"files": file_dict})

        return Schema(
            datashape=None,
            dtype=None,
            shape=None,
            npartitions=len(file_dict),
            extra_metadata=metadata,
        )

    def read_partition(self, file, **kwargs):
        """
        Read a specific sheet from the list of sheets
        file
            filename without extension
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        ext = self.metadata.get("ext", self._ext)
        srctype = self.metadata.get("type")
        if srctype == "file":
            return pd.read_hdf(self.metadata.get("src"))
        filename = "{file}.{ext}".format(file=file, ext=ext)
        if filename in self.metadata.get("files", []):
            filepath = self.metadata["files"][filename]
            return pd.read_hdf(filepath)
        else:
            return "No such HDF file"

    def read(self, **kwargs):
        """
        Read all sheets into a single dataframe.
        Sheetname is added as a column
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        if self.metadata.get("type") == "file":
            return pd.read_hdf(self.metadata.get("src"))
        else:
            return "The datasource is a directory.Use the read_partition method to read a specific file."

    def _close(self):
        print("Not implemented")

def twin_plot(data, y_axis, x_axis="timestamp"):
    """
    Create a bokeh plot with twin axes
    """
    from bokeh.plotting import figure
    from bokeh.models import LinearAxis, Range1d

    TOOLTIPS = [("datetime", "@x{%F %H:%M}"), ("value", "$y{0.00}")]

    y1, y2 = y_axis[0], y_axis[1]
    h0 = data[y1].max()
    l0 = data[y1].min()
    h1 = data[y2].max()
    l1 = data[y2].min()
    p = figure(
        x_axis_type="datetime",
        y_range=(l0, h0),
        tooltips=TOOLTIPS,
        height=240,
        width=600,
    )
    p.line(data[x_axis].values, data[y1].values, color="red", legend=y1)
    p.extra_y_ranges = {"foo": Range1d(l1, h1)}
    p.line(data[x_axis], data[y2].values, color="blue", y_range_name="foo", legend=y2)
    p.add_layout(LinearAxis(y_range_name="foo", axis_label=y2), "left")
    p.hover.formatters = {"x": "datetime"}
    p.legend.location = "top_center"
    p.legend.click_policy = "hide"
    return p

def conditional(data, c1, c2, out=None):
    """
    Create a conditional probability table with counts
    data
        dataframe
    c1
        condition as string
    c2
        list of conditions as strings
    out
        output format. If None, counts are returned.
        If a function is passed, it is applied to
        each of the conditions and the return value
        of the function is stored for each condition.
        The function should have a single argument that
        takes a dataframe as an input.

    returns a dictionary with the conditions and
    the counts or the return value of each of the conditions
    Note
    ----
    1. The dataframe is queried with c1 and each of the conditions
    in c2 are evaluated based on this result.
    2. All conditions are evaluated using `df.query`
    3. The condition strings should be valid columns in the dataframe
    4. The function passed should have a single argument, the dataframe.
    """
    dct = {}
    if out is None:
        out = len
    df = data.query(c1)
    dct[c1] = out(df)
    for c in c2:
        dct[c] = out(df.query(c))
    return dct

class Catalog:
    """
    A intake catalog creator
    The catalog is created in the following manner
        1. All files in the root directory are considered to
        be separate files and loaded as such.
        2. All directories and subdirectories inside the root
        directory are considered to be separate data sources
        3. Each of the files are matched against the extension
        name and the corresponding mapper
        4. Files inside a directory are randomly selected and the
        file type is determined for the entire directory.
        **It's assumed that all files inside any sub-directories
        are of the same file type**
    """

    def __init__(self, directory):
        """
        directory
            directory to search for files
        """
        self._directory = directory
        """
        All files in the below directories are added
        individually as a data source
        """
        self._file_dirs = ["files"]
        """
        **filetypes** is a dictionary with the file type as
        key and a sub-dictionary with driver and extensions
        as keys and the corresponding driver and extensions
        as values.
        It is a logical structure that maps a file type to
        its intake driver since each file type can have more
        than one extension. This dictionary is looped to get
        the self._mappers for each extension.
        **Assumed each filetype has a single driver but more than
        one extension**
        """
        filetypes = {
            "excel": {
                "driver": "fastbt.experimental.ExcelSource",
                "extensions": ["xls", "xlsx"],
            },
            "csv": {
                "driver": "intake.source.csv.CSVSource",
                "extensions": ["csv", "txt"],
            },
            "hdf": {
                "driver": "fastbt.experimental.HDFSource",
                "extensions": ["h5", "hdf5"],
            },
        }
        self._mappers = {}
        for k, v in filetypes.items():
            for ext in v["extensions"]:
                self._mappers[ext] = v["driver"]

    def generate_catalog(self):
        """
        Generate catalog
        #TO DO#
        1. Replace multiple dots with underscores in filenames
        """
        dct = {}
        dct["sources"] = {}
        src = dct["sources"]

        def metadata():
            """
            metadata generation for the file; has access to
            all variables inside the parent function
            """
            return {
                "args": {first_arg: os.path.join(dirpath, file)},
                "driver": self._mappers[ext],
                "description": "",
                "metadata": {"extension": ext, "mode": mode},
            }

        for dirpath, dirnames, filenames in os.walk(self._directory):
            dirname = dirpath.split("/")[-1]  #
            if dirname in self._file_dirs:
                mode = "file"
                for file in filenames:
                    ext = file.split(".")[-1]
                    if "csv" in ext:
                        first_arg = "urlpath"
                    else:
                        first_arg = "datapath"
                    if ext in self._mappers:
                        src[file.split(".")[0]] = metadata()
            else:
                mode = "dir"
                # If the directory has any files
                if len(filenames) > 0:
                    # Check the extension of the first file in directory
                    ext = filenames[0].split(".")[-1]
                    file = "*"
                    if "csv" in ext:
                        first_arg = "urlpath"
                        file = "*"
                    else:
                        first_arg = "datapath"
                        file = ""
                    if ext in self._mappers:
                        src[dirname] = metadata()
        return dct

def candlestick_plot(data):
    """
    return a bokeh candlestick plot
    data
        dataframe with open,high,low and close columns
    Note
    -----
    Prototype copied from the below link
    https://bokeh.pydata.org/en/latest/docs/gallery/candlestick.html
    """
    from math import pi
    from bokeh.plotting import figure
    from bokeh.models import ColumnDataSource

    df = data.copy()
    df["date"] = pd.to_datetime(df["date"])
    df["color"] = ["green" if x > y else "red" for (x, y) in zip(df.close, df.open)]
    source = ColumnDataSource()
    source.data = source.from_df(df)
    w = 10 * 60 * 1000  # half day in ms
    TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
    p = figure(
        x_axis_type="datetime",
        tools=TOOLS,
        title="Candlestick",
        plot_width=800,
        tooltips=[
            ("date", "@date{%F}"),
            ("open", "@open{0}"),
            ("high", "@high{0}"),
            ("low", "@low{0}"),
            ("close", "@close{0}"),
        ],
    )
    p.hover.formatters = {"date": "datetime"}
    p.xaxis.major_label_orientation = pi / 4
    p.grid.grid_line_alpha = 0.3
    p.segment("date", "high", "date", "low", color="black", source=source)
    p.vbar(
        "date",
        w,
        "open",
        "close",
        fill_color="color",
        line_color="black",
        source=source,
    )
    return p, source

def calendar_plot(data, field="ret"):
    """
    return a calendar plot
    data
        dataframe with year and month columns
    field
        field to plot values
    Note
    -----
    Prototype copied from bokeh gallery
    https://bokeh.pydata.org/en/latest/docs/gallery/unemployment.html
    """
    from math import pi
    from bokeh.models import (
        LinearColorMapper,
        BasicTicker,
        PrintfTickFormatter,
        ColorBar,
    )
    from bokeh.plotting import figure
    from bokeh.palettes import Spectral

    df = data.copy()
    # Type conversion
    df["year"] = df.year.astype("str")
    df["month"] = df.month.astype("str")
    years = list(df.year.unique())
    months = [str(x) for x in range(12, 0, -1)]
    colors = list(reversed(Spectral[6]))
    mapper = LinearColorMapper(
        palette=colors, low=df[field].min(), high=df[field].max()
    )
    TOOLS = "hover,save,pan,box_zoom,reset,wheel_zoom"
    p = figure(
        title="Calendar Plot",
        x_range=years,
        y_range=list(reversed(months)),
        x_axis_location="above",
        plot_width=1000,
        plot_height=600,
        tools=TOOLS,
        toolbar_location="below",
        tooltips=[("date", "@year-@month"), ("return", "@{}".format(field))],
    )
    # Axis settings
    p.grid.grid_line_color = None
    p.axis.axis_line_color = None
    p.axis.major_tick_line_color = None
    p.axis.major_label_text_font_size = "10pt"
    p.axis.major_label_standoff = 0
    p.xaxis.major_label_orientation = pi / 3

    p.rect(
        x="year",
        y="month",
        width=1,
        height=1,
        source=df,
        fill_color={"field": field, "transform": mapper},
        line_color="black",
        line_width=0.5,
        line_alpha=0.3,
    )

    color_bar = ColorBar(
        color_mapper=mapper,
        major_label_text_font_size="8pt",
        ticker=BasicTicker(desired_num_ticks=len(colors)),
        formatter=PrintfTickFormatter(format="%d%%"),
        label_standoff=6,
        border_line_color=None,
        location=(0, 0),
    )
    p.add_layout(color_bar, "right")
    return p

def summary_plot(data):
    """
    Given a dataframe, create a widget with condition and list
    of columns to provide summary data
    data
        dataframe
    Note
    -----
     * Condition is a valid eval string supported by pandas
     * Multiple columns can be selected
     * Summary statistics provided by the describe method

    """
    from bokeh.models import (
        TextInput,
        Button,
        MultiSelect,
        PreText,
    )
    from bokeh.layouts import layout

    df = data.copy()

    def document(doc):
        condition = TextInput(title="Enter your condition")
        col = MultiSelect(title="Select the columns", options=list(df.columns))
        button = Button(label="Update")
        pre = PreText()
        pre.text = condition.value

        def update():
            cond = condition.value
            cols = col.value
            text = df.query(cond)[cols].describe()
            pre.text = str(text)

        button.on_click(update)

        l = layout([[[condition, button], col], pre])
        doc.add_root(l)

    return document

def slider_plot(data, cols):
    """
    Given a dataframe, create a widget with range slider
    data
        dataframe
    cols
        columns for which sliders are to be created
    """
    from bokeh.models import (
        Button,
        RangeSlider,
        MultiSelect,
        PreText,
    )
    from bokeh.layouts import column, layout

    df = data.copy()

    def document(doc):
        sliders = []
        pre = PreText(text="something")
        select = MultiSelect(title="Select the columns", options=list(df.columns))
        button = Button(label="Update")
        for col in cols:
            MIN, MAX = df[col].min(), df[col].max()
            STEP = (MAX - MIN) / 100
            slider = RangeSlider(
                start=MIN, end=MAX, step=STEP, value=(MIN, MAX), title=col
            )
            sliders.append(slider)

        def update():
            values = []
            txt = "({col} > {low}) & ({col} < {high})"
            for slider in sliders:
                low, high = slider.value
                low, high = float(low), float(high)
                formula = txt.format(col=slider.title, low=low, high=high)
                values.append(formula)
            q = "&".join(values)
            summary_cols = select.value
            text = df.query(q)[summary_cols].describe()
            pre.text = str(text)

        button.on_click(update)

        l = layout([column(sliders), [select, button]], pre)
        doc.add_root(l)

    return document

def run_simulation(data, size=0.3, num=1000, column=None, function=np.mean):
    """
    run a simulation on the given data by drawing repeated samples
    and evaluating a metric
    data
        a pandas dataframe
    size
        size of the sample - float/int
        if size is a float and less than , it is considered a percentage
        in case of a num it is considered as the number of samples to take
    num
        number of simulations to be run
    column
        column for which sample is to be taken.
        If None, the entire dataframe is sampled and this is considerably slow
    function
        function to be run on the column - string/function
        default mean

    returns a pandas Series with the result for each sample taken
    """
    choice = np.random.choice
    if size < 1:
        N = int(len(data) * size)
    else:
        N = size
    collect = []
    if column is None:
        # Take sample on the whole dataframe
        for i in range(num):
            sample = function(data.sample(N))
            collect.append(sample)
    else:
        # Take sample on a particular column
        # considerably faster since this is a numpy version
        col = data[column].values
        for i in range(num):
            sample = function(choice(col, N))
            collect.append(sample)
    return pd.Series(collect)

def generate_parameters(dict_of_parameters):
    """
    Generate a list of parameters for backtest function
    dict_of_parameters
        dictionary of parameters in the given format
    Note
    -----
    Only one level of nested dictionary is parsed
    """
    from itertools import product

    d = dict_of_parameters.copy()
    lst = []

    def simple(dictionary, updt):
        # An inner function for ugly lookup
        # TO DO: Think of better way -> recursion
        listed = []
        for k, v in dictionary.items():
            if isinstance(v, list):
                listed.append([{k: val} for val in v])
            elif isinstance(v, str):
                listed.append([{k: v}])
        temp = list(product(*listed))
        lstk = []
        for tp in temp:
            temp_dict = updt.copy()
            for t in tp:
                temp_dict.update(t)
            lstk.append(temp_dict)
        return lstk

    for k, v in d.items():
        if isinstance(v, list):
            lst.append([{k: val} for val in v])
        elif isinstance(v, str):
            lst.append([{k: v}])
        elif isinstance(v, dict):
            dict_lst = []
            for k1, v1 in v.items():
                lst2 = simple(d[k][k1], {k: k1})
                dict_lst.extend(lst2)
            lst.append(dict_lst)
    return lst

@njit
def traverse(high, low, points):
    """
    See whether the price points are hit in the given order
    high
        high values as numpy array
    low
        low values as numpy array
    points
        list or numpy array of values to check
    returns a 4-tuple with
      1. an array indicating whether the values in
      the points list is hit; 1 indicates the value is hit
      2. an array of timesteps indicating when the
      value is hit; this is just the iteration cycle
      3. high value when the last point is hit or the
      end of the iteration cycle if the point is not hit
      4. low value when the last point is hit or the
      end of the iteration cycle if the point is not hit
    Note
    ----
    1. This function checks whether the points are hit
    in the given order.
    2. A point is considered hit if it is between high and
    low value
    3. high and low values are considered to be array of equal
    size with the value of high always greater than or equal
    to the value of low at every timestep
    """
    j = 0
    price = points[j]
    hit = np.zeros(len(points))
    timesteps = np.zeros(len(points))
    for i in range(len(high)):
        if low[i] < price < high[i]:
            hit[j] = 1
            timesteps[j] = i
            j += 1
            if j == len(points):
                break
            else:
                price = points[j]
    return (hit, timesteps, high[i], low[i])

class Strategy:
    """
    An automated strategy implementation class
    """

    def __init__(self, **kwargs):
        """
        keyword arguments
        result_cols
            column names for any extra value emitted
            from the tradebook function. The first 4
            columns are automatically named entry_time,
            entry_price, exit_time and exit_price and
            must correspond accordingly
        """
        self.datas = []
        self.result_cols = kwargs.pop("result_cols", None)

    @staticmethod
    def tradebook(open, high, low, close, **kwargs):
        """
        Numpy tradebook implementation with logic.
        To be implemented for each case
        Note
        -----
        1) Should always return an array of values
        2) The array must be a one-dimensional array
        3) The first 4 values in the array must be
        entry time, entry price, exit time and exit price
        4) It is assumed only one trade is done each day
        5) For multiple trades, modify the result function
        """
        raise NotImplementedError

    def _agged(self, times, data=None):
        """
        Get the aggregated data for the given times
        data
            intraday dataframe. If None, self.datas[0] is picked
        times
            list of 2-tuples with start and end times
        Note
        -----
        1) For each time tuple given in list, data is aggregated
        in OHLC form and added to a dataframe
        2) Columns are numbered based on the order of the times
        list so that the first tuple is named 0, the next 1 and so on
        3) Expect a date and timestamp column in the list
        """
        from fastbt.utils import recursive_merge

        agged = {"open": "first", "high": "max", "low": "min", "close": "last"}
        dfs = []
        if not (data):
            data = self.datas[0]
        for i, (s, e) in enumerate(times):
            temp = (
                data.set_index("timestamp")
                .between_time(s, e)
                .groupby("date")
                .agg(agged)
            )
            columns = {
                x: "{x}{i}".format(x=x, i=i) for x in ["open", "high", "low", "close"]
            }
            temp.rename(columns=columns, inplace=True)
            dfs.append(temp)
        return recursive_merge(dfs, on=["date"]).reset_index()

    def _each_day(self, data=None, cols=None, **kwargs):
        """
        Runs the tradebook function on each day.
        data
            dataframe on which tradebook simulation is run.
            In case of None, the last item in self.datas is used
            The dataframe must have a date column and it must
            be sorted by timestamp
        cols
            columns to be passed to the tradebook function.
            The columns should be argument names in the
            tradebook function
        kwargs
            keyword arguments would be passed to the tradebook function
        """
        if not (data):
            data = self.datas[-1]  # the last appended data
        grouped = data.groupby("date")
        if not (cols):
            cols = ["open", "high", "low", "close"]
        tradebook = self.tradebook

        def tb(x):
            dct = {c: x[c].values for c in cols}
            kwargs.update(dct)
            return tradebook(**kwargs)

        res = grouped.apply(tb)
        return res

    def result(self, **kwargs):
        """
        Result of the strategy.
        The output of the tradebook function is the input to
        this function. Converts the array into a dataframe
        and adds some useful columns
        """
        tmp = self._each_day(**kwargs)

        def get_column_names():
            """
            Get column names
            """
            cols = ["entry_time", "entry_price", "exit_time", "exit_price"]
            if self.result_cols:
                print(cols + self.result_cols)
                return cols + self.result_cols
            else:
                L = len(tmp.iloc[0])
                return cols + [f"col{i}" for i in range(4, L)]

        res = pd.DataFrame(
            tmp.values.tolist(), index=tmp.index, columns=get_column_names()
        )
        res["year"] = res.index.year
        res["profit"] = res.eval("exit_price-entry_price")
        res["cum_p"] = res.profit.cumsum()
        res["max_p"] = res.cum_p.expanding().max()
        return res

    def metric(self):
        """
        Default metric to use
        """
        return

    def plot(self, data):
        """
        A general plotting function
        """
        data[["cum_p", "max_p"]].plot()
        plt.figure()
        data.groupby("year").profit.sum().plot.bar()

    def simulator(self, **kwargs):
        """
        Run simulation and return results
        kwargs
            keyword arguments where argument is the name of
            the variable and values as list the variable takes.
            The keyword variable is passed to the tradebook
            function apart from OHLC values
        """
        pass

    def run(self, plotting=True, **kwargs):
        """
        Run the entire process
        """
        res = self.result(**kwargs)
        if plotting:
            self.plot(res)
        return res  # For further analysis

def advances(data, date="date", column=None, out="advances"):
    """
    Get the advances by day
    data
        dataframe with the necessary columns
    date
        date column in the dataframe to group by
    column
        column on which ratio would be calculated, usually a returns column
        If column is not available, the close/open return is calculated
    out
        output format one of advances/declines/difference/ratio/all
        all returns everything
    """
    data = data.copy()
    if column is None:
        data["pret"] = data.eval("(close/open)-1")
        column = "pret"
    col = "is_" + column
    data[col] = data.eval(f"{column}>0") + 0
    data2 = data.groupby([date, col]).size().unstack().fillna(0).astype(int)
    data2.columns = ["declines", "advances"]
    data2["difference"] = data2.eval("advances-declines")
    data2["ratio"] = data2.eval("advances/declines")
    if out == "all":
        return data2
    else:
        return data2[out]

class CodeGenerator:
    def __init__(self, name, blocks=None):
        self._name = name
        self._struct = []
        self._blocks = {}
        self._block_names = []

    @property
    def name(self):
        return self._name

    def add_block(self, name, indent=False, n=4):
        """
        Add a code block
        name
            name of the code block
        indent
            whether to apply indentation to the code
            only one level of indentation is added
        Note
        -----
        1. name should be a key in the blocks dictionary
        2. indentation is provided with 4 spaces

        """
        self._block_names.append(name)
        if indent:
            txt = "{{" + str(name) + " | indent(" + str(n) + ", first=True)}}"
        else:
            txt = "{{" + str(name) + "}}"
        self._struct.append(txt)

    def add_text(self, text):
        """
        Add text
        text
            Text to add
        the given text is added without any variable interpolation
        """
        self._struct.append(text)

    def add_code_block(self, name, block):
        """
        Add a code block to the existing blocks dictionary
        name
            name of the block
        block
            actual code
        If name is already in the blocks dictionary,
        it would be overwritten
        """
        self._blocks[name] = block

    def clear(self):
        """
        Clear everything in the existing structure
        """
        self._struct = []
        self._block_names = []

    def generate_code(self):
        from jinja2 import Template

        code = "\n".join(self._struct)
        template = Template(code)
        substitution = {b: self._blocks.get(b) for b in self._block_names}
        return template.render(**substitution)

    def save_file(self, filename=None):
        if not (filename):
            filename = self.name + ".py"
        code = self.generate_code()
        with open(filename, "w") as f:
            f.write(code)

def renko_plot(data, bricks_col="brick"):
    """
    Draw a renko plot from the given dataframe
    data
        dataframe containing renko data
    bricks_col
        name of the bricks column; default brick
    Note
    -----
    Brick size is calculated from bricks column automatically
    """
    from bokeh.plotting import figure

    data = data.copy()
    brick_size = abs(data[bricks_col].iloc[0] - data[bricks_col].iloc[1])
    data["left"] = range(len(data))
    data["right"] = data.left.values + 1
    data["top"] = data[bricks_col].values
    data["bottom"] = data.top + brick_size
    data["move"] = (data.brick > data.brick.shift(1)) + 0
    data["color"] = ["green" if x == 1 else "red" for x in data.move.values]
    p = figure(title="Renko chart")
    p.quad(
        top="top",
        bottom="bottom",
        left="left",
        right="right",
        color="color",
        source=data,
    )
    return p

class DayTrading:
    def __init__(self, data=None, interval=None, tradebook=None, tradebook_args={}):
        """
        arguments to be passed to init
        data
            primary dataframe
            this is the dataframe on which all work would be done
        interval
            interval as pandas dataframe
        tradebook
            a valid tradebook function
        tradebook_args
            extra argument to be passed to tradebook
        """
        self._interval = interval
        self._data = data
        self._sources = {}
        self._tradebook = tradebook
        self._tradebook_args = tradebook_args
        import pyfolio as pf

        self.pf = pf

    @property
    def data(self):
        return self._data

    @staticmethod
    def agged(data, interval="5min", column_name="timestamp"):
        return (
            data.set_index(column_name)
            .resample(interval, label="right")
            .agg({"open": "first", "high": "max", "low": "min", "close": "last"})
        )

    def _by_day(self):
        """
        Run the function on each day
        """
        func_spec = inspect.getfullargspec(self._tradebook)
        columns = self.data.columns

        def f(data):
            kwargs = {}
            for arg in func_spec.args:
                if arg in columns:
                    if arg in func_spec.annotations.keys():
                        kwargs[arg] = data[arg].values[0]
                    else:
                        kwargs[arg] = data[arg].values
            kwargs.update(self._tradebook_args)
            return self._tradebook(**kwargs)

        grouped = self.data.groupby("date")
        tbs = grouped.apply(f)
        trades = []
        for v in tbs.values:
            trades.extend(v.all_trades)
        return trades
        # This is excess to be corrected
        all_trades = pd.DataFrame(trades)
        all_trades["date"] = pd.to_datetime(all_trades.ts.dt.date)
        all_trades["value"] = all_trades.eval("price*qty*-1")
        return all_trades.tail()

    def _convert_to_legs(self, result=None):
        """
        Convert trades to daily legs for better summary
        result
            a tradebook with alternate buy and sell trades
        """
        if result is None:
            result = self._result
        x = range(0, len(result), 2)
        y = range(1, len(result), 2)
        trds = []
        for m, n in zip(x, y):
            a = result[m]
            b = result[n]
            d = {}
            d["symbol"] = a["symbol"]
            d["order"] = a["order"]
            d["entry_time"] = a["ts"]
            d["entry_price"] = a["price"]
            d["qty"] = a["qty"]
            d["exit_time"] = b["ts"]
            d["exit_price"] = b["price"]
            trds.append(d)
        trds = pd.DataFrame(trds)
        trds["entry_time"] = pd.to_datetime(trds.entry_time)
        trds["exit_time"] = pd.to_datetime(trds.exit_time)
        trds["hour"] = trds.entry_time.dt.hour
        trds["date"] = pd.to_datetime(trds.entry_time.dt.date)
        trds["pnl"] = trds.eval("(exit_price-entry_price)*qty")
        return trds

    @property
    def summary(self):
        return self._summary

    def add_source(self, name: str, data: str):
        """
        adds a data source to the existing sources.
        you could access this as an attribute
        name
            the name of the data source
        data
            actual data as a pandas dataframe
        Note
        -----
        This is just a convenient function to
        add a datasource as an attribute
        """
        self._sources[name] = data
        setattr(self, name, data)

    def perf_stats(self, cost=0, capital=1000):
        returns = (self.summary.set_index("date").pnl - cost) / capital
        return self.pf.timeseries.perf_stats(returns)

    def drawdown_table(self, cost=0, capital=1000):
        returns = (self.summary.set_index("date").pnl - cost) / capital
        return self.pf.timeseries.gen_drawdown_table(returns)

    def run(self):
        print("Started running the program")
        self._result = self._by_day()
        summary = self._convert_to_legs()
        self._summary = summary

def single_filter(frame, col1, col2, func=np.mean):
    """
    Create a single filter and returns results as a dictionary
    frame
        dataframe
    col1
        column on which the filter is to be applied
    col2
        list of columns with each column being a categorical
    func
        arbitary function to be applied on each group
    """
    collect = {}
    for col in col2:
        val = frame.groupby(col)[col1].apply(func)
        collect[col] = val
    return collect

@njit
def clean_ticks(price, threshold=10):
    """
    Clean out of sample ticks
    """
    length = len(price)
    arr = np.zeros(length)
    s = price[0]
    dropped = 0
    nobs = 0
    for i in np.arange(1, length):
        if np.abs(price[i] - s) < threshold:
            arr[i] = price[i]
            s = price[i]
            nobs = 0
        else:
            nobs += 1
            dropped += 1
            if nobs > 10:
                s = price[i]
    return arr[arr > 0]

class WalkForward:
    def __init__(self, data, lb=120, rb=30, factor=None, column=None):
        from collections import defaultdict

        self.data = data
        self.lb = lb
        self.rb = rb
        self._splits = defaultdict(list)
        self._factor = factor
        self._column = column
        self._results = []
        self.conf = []
        self.forward = []

    @property
    def factor(self):
        return self._factor

    @property
    def column(self):
        return self._column

    def get_splits(self):
        return self._splits

    def _generate_splits(self):
        lb, rb = self.lb, self.rb
        indexes = range(lb, len(self.data), rb)
        for index in indexes:
            train = self.data.iloc[index - lb : index]
            test = self.data.iloc[index : index + rb]
            self._splits["train"].append(train)
            self._splits["test"].append(test)

    def set_factor(self, factor):
        """
        The factor should be a categorical column in data
        You can also provide a list of category columns
        """
        if factor in self.data.columns:
            self._factor = factor
            return self._factor
        else:
            return "Factor not found"

    def set_column(self, column):
        """
        The column to test
        """
        if column in self.data.columns:
            self._column = column
            return self._column
        else:
            return "Column not found"

    def run(self):
        """
        Run all tests
        """
        fac = self.factor
        col = self.column
        self._generate_splits()
        splits = self.get_splits()
        for train in splits["train"]:
            t1 = train.groupby(fac)[col].agg(["size", "mean"]).to_dict("index")
            self._results.append(t1)
        for test in splits["test"]:
            t2 = test.groupby(fac)[col].agg(["size", "mean"]).to_dict("index")
            self.forward.append(t2)

    def run_conf(self):
        """
        Run confidence test
        """
        print("Running conf")
        train = self.get_splits()["train"]
        results = self._results
        for data, res in zip(train, results):
            tup = []
            for k, v in res.items():
                s = run_simulation(data, size=v["size"], column=self.column)
                conf = len(s[s > v["mean"]]) / 1000
                tup.append((k, v["mean"], conf))
            self.conf.append(tup)


import numpy as np
import matplotlib.pyplot as plt

data_x = np.array([1,2,3])
data_y = np.array([3,5,7])

def E(array):
    return np.average(array)

def Linear():
    x = data_x
    y = data_y

    a = (E(x*y) - E(x) * E(y)) / (E(x*x) - E(x) * E(x))
    b = E(y) - a * E(x)
    print(a,b)

Linear()


# -*- coding: utf-8 -*-
"""
Created on Sat May 01 6:39:27 2021

@author: Johan Monster

Sandbox for testing Cube Projector functionality

"""

#%% ==== Import dependencies ====

import numpy as np
from copy import deepcopy
from numpy import sin, cos
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import mpl_toolkits.mplot3d as mp3d

from cp_vertex import Vertex
from cp_face import Face
from cp_geometry import Geometry
from cp_vector import Vector
from cp_frame import Frame
from cp_utilities import d2r#, r2d
from cp_plotting import plot_global_tripod, plot_frame, \
    plot_vertex, plot_face

#%% ==== Manipulate objects ====

frame1 = Frame()
frame1.translate(1,1,1)

frame2 = Frame()

# p1 = Vertex(0,0,0,frame1)
# p2 = Vertex(1,0,0)
# p3 = Vertex(1,1,0)
# p4 = Vertex(0,1,0)
# p5 = Vertex(0,1,1)
# p6 = Vertex(0,0,1)

# f1 = Face(p1, p2, p3, p4, frame1)
# f2 = Face(p1, p4, p5, p6, frame1)

frame1.readout()

f1 = Face(Vertex(0,0,0),
          Vertex(1,0,0),
          Vertex(1,1,0),
          Vertex(0,1,0),
          frame1)

frame1.readout()

del(f1)

frame1.readout()

#%% ==== Plotting ====

# Toggle plotting functionality:
if False:

    # Setting up the plot:
    fig = plt.figure(figsize=(8, 8))
    ax = mp3d.Axes3D(fig)

    """ TO CHANGE THE DEFAULT CAMERA VIEW, CHANGE THESE: """
    ax.view_init(elev=20, azim=-60)

    # Setting up the axes object
    ax.set_title("Wireframe visualization.")

    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)
    ax.set_zlim(-2, 2)

    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('z')

    # Plotting the global XYZ tripod
    plot_global_tripod(ax)

    # ==== Plot custom objects ====

    for vertex in [p1, p2, p3, p4, p5, p6]:
        plot_vertex(ax, vertex)

    for face in [f1, f2]:
        plot_face(ax, face)

    plot_frame(ax, frame1)

    # =============================

    plt.show()


from fastapi_users.authentication import BearerTransport, AuthenticationBackend, JWTStrategy

bearer_transport = BearerTransport(tokenUrl="auth/jwt/login")

SECRET = "SECRET"

def get_jwt_strategy() -> JWTStrategy:
    return JWTStrategy(secret=SECRET, lifetime_seconds=3600)

auth_backend = AuthenticationBackend(
    name="jwt",
    transport=bearer_transport,
    get_strategy=get_jwt_strategy,
)