import torch
import torch.optim
from collections import OrderedDict
import torch.nn as nn

class Net(nn.Module):
    def __init__(self, seq_net, name='MLP', activation=torch.tanh):
        super().__init__()
        self.features = OrderedDict()
        for i in range(len(seq_net) - 1):
            self.features['{}_{}'.format(name, i)] = nn.Linear(seq_net[i], seq_net[i + 1], bias=True)
        self.features = nn.ModuleDict(self.features)
        self.active = activation

        # initial_bias

        for m in self.modules():
            if isinstance(m, nn.Linear):
                nn.init.constant_(m.bias, 0)
                nn.init.xavier_normal_(m.weight, gain=1.0) #ini2
                # torch.nn.init.uniform_(m.weight, a=-0.01, b=0.01) #3
                # torch.nn.init.uniform_(m.weight, a=-0.1, b=0.1) #4
                # torch.nn.init.normal_(m.weight, mean=0.0, std=0.01) #5
                # torch.nn.init.normal_(m.weight, mean=0.0, std=0.1) #6

    def forward(self, x):
        # x = x.view(-1, 2)
        length = len(self.features)
        i = 0
        for name, layer in self.features.items():
            x = layer(x)
            if i == length - 1: break
            i += 1
            x = self.active(x)
        return x


from enigma import *

r1 = Rotor("VEADTQRWUFZNLHYPXOGKJIMCSB", 1)
r2 = Rotor("WNYPVJXTOAMQIZKSRFUHGCEDBL", 2)
r3 = Rotor("DJYPKQNOZLMGIHFETRVCBXSWAU", 3)
reflector = Reflector("EJMZALYXVBWFCRQUONTSPIKHGD")
machine = Machine([r1, r2, r3], reflector)
x = machine.encipher("Hello Mr.AhmadZade!")
print(x)
print(machine.decipher(x))


import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
import Myfun
Myfun.pyplot_中文()

# Load the diabetes dataset
diabetes = datasets.load_diabetes()

# Use only one feature
diabetes_X = diabetes.data[:, np.newaxis, 2]  # BMI

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % regr.score(diabetes_X_test, diabetes_y_test))

# Plot outputs

plt.scatter(diabetes_X_test, diabetes_y_test,  color='black')
plt.plot(diabetes_X_test, regr.predict(diabetes_X_test), color='blue',
         linewidth=3)
plt.xlabel("BMI")
plt.ylabel("Target 一年後的患病指數")
# plt.xticks(())
# plt.yticks(())

plt.show()

Regression回歸分析/糖尿病/回歸分析找出BMI與糖尿病關係.py

回歸分析找出BMI與糖尿病關係.py


# -*- coding: utf-8 -*-
import sys
import numpy as np
import matplotlib.pyplot as plt
import scipy

sys.path.append('../')
from mlFunc import *
from validators import *
from classifiers import *
from prettytable import PrettyTable
from Classifiers.GMM import GMM_Full
import scipy.stats as stats
from validation.validation_GMM import kfold_GMM

def validation_GMM(title, pi, GMM_llrs, LTE):
    GMM_llrs = np.hstack(GMM_llrs)
    llrs_tot = compute_min_DCF(GMM_llrs, LTE, pi, 1, 1)
    llrs_tot_act = compute_act_DCF(GMM_llrs, LTE, pi, 1, 1)
    llrs_tot_xvd = compute_act_DCF(GMM_llrs, LTE, pi, 1, 1, -np.log(pi / (1 - pi)))

    # t = PrettyTable(["Type", "minDCF"])
    # t.title = title
    # t.add_row(["GMM_EM", round(llrs_tot, 3)])
    # print(t)
    return round(llrs_tot, 3), round(llrs_tot_act, 3), round(llrs_tot_xvd, 3)

def ll_GMM(D, L, Dte, Lte, llr, cov, comp):
    # GMM_llrs, 'full', comp, i
    # CLASS PRIORS: WE CONSIDER A BALANCED APPLICATION

    # GMM MODELS
    # π = 0.5

    # optimal_m = 10
    optimal_comp = comp
    optimal_cov = cov
    optimal_alpha = 0.1
    optimal_psi = 0.01

    llr.extend(GMM_Full(D, Dte, L, optimal_alpha, 2 ** optimal_comp, optimal_cov).tolist())
    return llr

def plot_minDCF_GMM_eval(score_raw, score_gauss, title, components):
    labels = np.exp2(components).astype(int)
    raw_val, raw_eval = score_raw
    gauss_val, gauss_eval = score_gauss

    # for i in range(components):
    #     labels.append(2 ** (i+1))

    x = np.arange(len(labels))  # the label locations
    width = 0.1  # the width of the bars
    plt.bar(x - 0.15, raw_val, width, label='Raw [val]', edgecolor='black', color='tab:orange', alpha=0.5)
    plt.bar(x - 0.05, raw_eval, width, label='Raw [eval]', edgecolor='black', color='tab:orange')
    plt.bar(x + 0.05, gauss_val, width, label='Gauss [val]', edgecolor='black', color='r', alpha=0.5)
    plt.bar(x + 0.15, gauss_eval, width, label='Gauss [eval]', edgecolor='black', color='r')

    plt.xticks(x, labels)
    plt.ylabel("DCF")
    plt.title(title)
    plt.legend()
    plt.savefig('./images/GMM/' + title)
    plt.show()

def print_minDCF_tables(score_raw, score_gauss, components):
    types = ['full-cov', 'diag-cov', 'tied full-cov', 'tied diag-cov']

    header = ['']
    n_comp = len(components)
    print(np.shape(score_raw))
    print(score_raw)
    score_raw = np.reshape(np.hstack(score_raw), ((n_comp), 4)).T
    score_gauss = np.reshape(np.hstack(score_gauss), ((n_comp), 4)).T

    comp = np.exp2(components).astype(int).tolist()

    print(np.shape(score_raw))
    for i in comp:
        header.append(i)
    print(header)
    for i in range(len(types)):
        t1 = PrettyTable(header)

        t1.title = types[i]

        raw_full = score_raw[i].tolist()
        gauss_full = score_gauss[i].tolist()

        raw_full.insert(0, 'raw')
        gauss_full.insert(0, 'gaussianized')
        t1.add_row(raw_full)
        t1.add_row(gauss_full)
        print(t1)
        # plot_minDCF_GMM(score_raw[i].tolist(), score_gauss[i].tolist(), types[i], components)

def print_act_DCF_tables(score_raw, score_gauss, components):
    types = ['full-cov', 'diag-cov', 'tied full-cov', 'tied diag-cov']

    header = ['']
    n_comp = len(components)
    print(np.shape(score_raw))
    print(score_raw)
    score_raw = np.reshape(np.hstack(score_raw), ((n_comp), 4)).T
    score_gauss = np.reshape(np.hstack(score_gauss), ((n_comp), 4)).T

    comp = np.exp2(components).astype(int).tolist()

    print(np.shape(score_raw))
    for i in comp:
        header.append(i)
    print(header)
    for i in range(len(types)):
        t1 = PrettyTable(header)

        t1.title = types[i]

        raw_full = score_raw[i].tolist()
        gauss_full = score_gauss[i].tolist()

        raw_full.insert(0, 'raw')
        gauss_full.insert(0, 'gaussianized')
        t1.add_row(raw_full)
        t1.add_row(gauss_full)
        print(t1)

def plot_minDCF_GMM(score_raw, score_gauss, title, components):
    labels = np.exp2(components).astype(int)

    # for i in range(components):
    #     labels.append(2 ** (i+1))

    x = np.arange(len(labels))  # the label locations
    width = 0.35  # the width of the bars
    plt.bar(x - 0.2, score_raw, width, label='Raw')
    plt.bar(x + 0.2, score_gauss, width, label='Gaussianized')

    plt.xticks(x, labels)
    plt.ylabel("DCF")
    plt.title(title)
    plt.legend()
    plt.savefig('../images/GMM/' + title)
    plt.show()

def evaluation_GMM(DTR, LTR, DTE, LTE, pi, comp, zscore=False):
    GMM_llrs = []
    GMM_llrsn = []
    GMM_llrst = []
    GMM_llrsnt = []
    GMM_labels = []

    D = DTR
    L = LTR

    Dte = DTE
    Lte = LTE

    if (zscore):
        D = scipy.stats.zscore(D, axis=1)

    print("components: " + str(comp))
    GMM_labels = np.append(GMM_labels, Lte)
    GMM_labels = np.hstack(GMM_labels)

    # RAW DATA

    # full-cov
    GMM_llrs = ll_GMM(D, L, Dte, Lte, GMM_llrs, 'full', comp)

    # diag-cov
    GMM_llrsn = ll_GMM(D, L, Dte, Lte, GMM_llrsn, 'diag', comp)

    # full-cov tied
    GMM_llrst = ll_GMM(D, L, Dte, Lte, GMM_llrst, 'tied_full', comp)

    # diag-cov tied
    GMM_llrsnt = ll_GMM(D, L, Dte, Lte, GMM_llrsnt, 'tied_diag', comp)

    llrs_tot_min, llrs_tot_act, llrs_tot_xvd = validation_GMM("GMM full", pi, GMM_llrs, GMM_labels)
    llrsn_tot_min, llrsn_tot_act, llrsn_tot_xvd = validation_GMM("GMM diag", pi, GMM_llrsn, GMM_labels)
    llrst_tot_min, llrst_tot_act, llrst_tot_xvd = validation_GMM("GMM tied full", pi, GMM_llrst, GMM_labels)
    llrsnt_tot_min, llrsnt_tot_act, llrsnt_tot_xvd = validation_GMM("GMM tied diag", pi, GMM_llrsnt, GMM_labels)

    llrs_min = [llrs_tot_min, llrsn_tot_min, llrst_tot_min, llrsnt_tot_min]
    llrs_act = [llrs_tot_act, llrsn_tot_act, llrst_tot_act, llrsnt_tot_act]
    llrs_xvd = [llrs_tot_xvd, llrsn_tot_xvd, llrst_tot_xvd, llrsnt_tot_xvd]
    return llrs_min, llrs_act, llrs_xvd, GMM_llrs, GMM_llrsn, GMM_llrst, GMM_llrsnt, GMM_labels

def bayes_error_min_act_plot_GMM(D, LTE, pi, title, ylim):
    p = numpy.linspace(-3, 3, 21)
    pylab.title(title)
    pylab.plot(p, bayes_error_plot(p, D, LTE, minCost=False), color='r', label='actDCF')
    pylab.plot(p, bayes_error_plot(p, D, LTE, minCost=True), 'r--', label='minDCF')
    pylab.plot(p, bayes_error_plot(p, D, LTE, minCost=False, th=-np.log(pi / (1 - pi))), color='y', label='theoretical')
    pylab.ylim(0, ylim)
    pylab.legend()
    pylab.savefig('../images/DCF_' + title + '.png')
    pylab.show()

def bayes_plot_bestGMM(title, pi, GMM_llrs, GMM_llrsn, GMM_llrst, GMM_llrsnt, GMM_labels):
    bayes_error_min_act_plot_GMM(GMM_llrs, GMM_labels, pi, 'GMM_full', 0.4)
    bayes_error_min_act_plot_GMM(GMM_llrsn, GMM_labels, pi, 'GMM_diag', 0.4)
    bayes_error_min_act_plot_GMM(GMM_llrst, GMM_labels, pi, 'GMM_tied', 0.4)
    bayes_error_min_act_plot_GMM(GMM_llrsnt, GMM_labels, pi, 'GMM_tied_diag', 0.4)

def evaluation_GMM_ncomp(typeof, DTR, LTR, DTE, LTE, pi, n, zscore=False):
    raw_min, raw_act, raw_x, GMM_llrs, GMM_llrsn, GMM_llrst, GMM_llrsnt, GMM_labels = evaluation_GMM(
        DTR, LTR, DTE, LTE, pi, n, zscore)
    print(raw_act, raw_x)

    types = ['full-cov', 'diag-cov', 'tied full-cov', 'tied diag-cov']
    t = PrettyTable(["", 'minDCF'])
    t.title = "GMM π=" + str(pi)
    for i in range(len(raw_min)):
        t.add_row([typeof + " " + types[i], raw_min[i]])
    print(t)

    return GMM_llrst
    # bayes_plot_bestGMM("prova", 0.5, GMM_llrs, GMM_llrsn, GMM_llrst, GMM_llrsnt, GMM_labels)
    # plot_ROC(GMM_llrs, GMM_labels, 'GMM_full2')
    # plot_ROC(GMM_llrsn, GMM_labels, 'GMM_diag2')
    # plot_ROC(GMM_llrst, GMM_labels, 'GMM_tied2')
    # plot_ROC(GMM_llrsnt, GMM_labels, 'GMM_tied_diag2')

def evaluation_scores_GMM_ncomp(typeof, DTR, LTR, DTE, LTE, pi, n, zscore=False):
    raw_min, raw_act, raw_x, GMM_llrs, GMM_llrsn, GMM_llrst, GMM_llrsnt, GMM_labels = evaluation_GMM(
        DTR, LTR, DTE, LTE, pi, n, zscore)

    return GMM_llrst

def experimental_GMM(DTR, LTR, DTE, LTE):
    score_raw_val = []
    score_gauss_val = []

    score_raw_eval = []
    score_gauss_eval = []
    DTE_gauss = gaussianize_features(DTR, DTE)
    DTR_gauss = gaussianize_features(DTR, DTR)
    # We'll train from 1 to 2^7 components

    # We'll train from 1 to 2^7 components
    componentsToTry = [1, 2, 3, 4, 5, 6, 7]
    for comp in componentsToTry:
        print('RAW DATA')
        raw_min, *_ = kfold_GMM(DTR, LTR, 0.5, comp, Gauss_flag=False)
        raw_eval, *_ = evaluation_GMM(DTR, LTR, DTE, LTE, 0.5, comp)
        score_raw_val.append(raw_min)
        score_raw_eval.append(raw_eval)

        print('GAUSSIANIZED')
        gauss_min, *_ = kfold_GMM(DTR, LTR, 0.5, comp, Gauss_flag=True)
        gauss_eval, *_ = evaluation_GMM(DTR_gauss, LTR, DTE_gauss, LTE, 0.5, comp)
        score_gauss_val.append(gauss_min)
        score_gauss_eval.append(gauss_eval)

    n_comp = len(componentsToTry)
    score_raw_val = np.reshape(np.hstack(score_raw_eval), (n_comp, 4)).T
    score_gauss_val = np.reshape(np.hstack(score_gauss_eval), (n_comp, 4)).T

    score_raw_eval = np.reshape(np.hstack(score_raw_eval), (n_comp, 4)).T
    score_gauss_eval = np.reshape(np.hstack(score_gauss_eval), (n_comp, 4)).T

    types = ['full-cov', 'diag-cov', 'tied full-cov', 'tied diag-cov']
    for i in range(len(types)):
        plot_minDCF_GMM_eval(
            [score_raw_val[i].tolist(), score_raw_eval[i].tolist()],
            [score_gauss_val[i].tolist(), score_gauss_eval[i].tolist()],
            types[i],
            componentsToTry)


#
# @lc app=leetcode.cn id=167 lang=python3
#
# [167] 两数之和 II - 输入有序数组
#

# @lc code=start
class Solution:
    def twoSum(self, numbers: List[int], target: int) -> List[int]:
        if len(numbers) <= 1:
            return []
        '''     # 循环过多
        point, i= 0, 1
        while point < len(numbers) and i < len(numbers):
            if numbers[point] + numbers[i] == target:
                return [point + 1, i + 1]
            elif numbers[point] + numbers[i] > target:
                point += 1
                i = point + 1
            else:
                i += 1
        return []
        '''
        # 用双指针，缩减搜索空间，关键是【i < j】
        i, j = 0, len(numbers)-1
        while i < j:
            now = numbers[i] + numbers[j]
            if now < target:
                i += 1
            elif now > target:
                j -= 1
            else:
                return [i + 1, j + 1]
        return []

# @lc code=end


"""This module implements the following classes:
    - AnimalsDataListView
"""

from PyQt5 import QtCore, QtGui, QtWidgets

from pigcel.gui.models.animals_data_model import AnimalsDataModel
from pigcel.gui.views.double_clickable_listview import DoubleClickableListView

class AnimalsDataListView(DoubleClickableListView):
    """This class implments the listview used to drop in animal names.
    """

    def keyPressEvent(self, event):
        """Event handler for keyboard interaction.

        Args:
            event (PyQt5.QtGui.QKeyEvent): the keyboard event
        """

        if event.key() == QtCore.Qt.Key_Delete:

            animals_data_model = self.model()
            if animals_data_model is None:
                return

            for sel_index in reversed(self.selectedIndexes()):
                animals_data_model.remove_workbook(sel_index.data(AnimalsDataModel.Workbook))
                if animals_data_model.rowCount() > 0:
                    index = animals_data_model.index(animals_data_model.rowCount()-1)
                    self.setCurrentIndex(index)

        else:
            super(AnimalsDataListView, self).keyPressEvent(event)


# -*- coding: utf-8 -*-
"""
@author: MetaMemeLord-
"""

def dfs(x,c,done):
	done[c] = False
	count = 0
	check = False
	for i in x[c]:
		if done[i]:
			check = True
			count += dfs(x,i,done)
	if not check:
		return 1
	return count+1

for i in range(int(input())):
	c,r,cl,cr = map(int,input().split())
	done = [True]*c
	x = [[] for i in range(c)]
	for p in range(r):
		i,j = map(int,input().split())
		x[i-1].append(j-1)
		x[j-1].append(i-1)
	sum = 0
	for i in range(c):
		if done[i]:
			val = 1000000000000000000000
			length = dfs(x,i,done)
			for j in range(1,length+1):
				val = min(j*cl + (length-j)*cr,val)
			sum += val
	print(sum)


import re

import docx
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill

class DocxTableReportParser:
    def __init__(self, docx_path):
        self.docx_path = docx_path
        self.workbook = Workbook()
        self.sheet = self.workbook.active
        self.tables = self._read_docx_table_all_elements_to_list(self.docx_path)
        # stili
        self.bold_font = Font(bold=True)
        self.center_aligned_text = Alignment(horizontal="center")
        self.light_gray_bg_color = PatternFill(start_color="D3D3D3", fill_type="solid")
        self.red_bg_color = PatternFill(start_color="FF0000", fill_type="solid")
        self.yellow_bg_color = PatternFill(start_color="FFFF00", fill_type="solid")
        self.cyan_bg_color = PatternFill(start_color="00FFFF", fill_type="solid")
        # dodamo prve dve vrstice
        self._add_first_line_template()
        self._add_second_line_template()

    def _append_rows(self, rows):
        for row in rows:
            self.sheet.append(row)

    def _get_class_id_version(self, table_id):
        data = self.tables[table_id][0][1].split(",")
        class_id = int(
            re.search(r"Class_id\s?=\s?(\d)", data[0], re.IGNORECASE).group(1)
        )
        version = int(re.search(r"version\s?=\s?(\d)", data[1], re.IGNORECASE).group(1))
        return class_id, version

    def _get_obis_code_daily(self, table_id):
        result = self.tables[table_id][2]
        result = [el.strip() for el in result]
        obis_code = (
            f"{result[1]}-{result[2]}:{result[3]}.{result[4]}.{result[5]}.{result[6]}"
        )
        return obis_code

    def _add_atributes_table(self, table_id):
        attributes = self.tables[table_id][3:]
        attributes_name = [
            [int(attr[0].split()[0].replace(".", "")), attr[0].split()[1]]
            for attr in attributes
        ]
        access_rights = []
        for ar in attributes:
            access_rights.append(
                [
                    el.replace("R/-", "Get_1")
                    .replace("R/W", "Get_1,Set_1")
                    .replace("-/- ", "")
                    for el in ar[2:]
                ]
            )
        final_list = []
        for an, ar in zip(attributes_name, access_rights):
            final_list.append([an[0], an[1], "", "", "", ar[0], ar[1], ar[3], ar[2]])
        self._append_rows(final_list)
        return len(attributes_name)

    def _read_docx_table_all_elements_to_list(self, document_path: str):
        doc = docx.Document(document_path)
        tables_doc = doc.tables
        final_tables = []
        for table in tables_doc:
            data = []
            for i, row in enumerate(table.rows):
                text = list((cell.text for cell in row.cells))
                data.append(text)
            final_tables.append(data)
        return final_tables

    def _add_first_line_template(self):
        """Add first line fix template."""
        rows = [
            [
                "",
                "Object/Attribute Name",
                "IC",
                "IC",
                "OBIS Object",
                "Access right",
                "Access right",
                "Access right",
                "Access right",
            ]
        ]

        self._append_rows(rows)

        # združimo celice
        self.sheet.merge_cells("C1:D1")
        self.sheet.merge_cells("F1:I1")
        # uredimo prvo vrstico
        for cell in self.sheet["1:1"]:
            cell.fill = self.light_gray_bg_color
            cell.alignment = self.center_aligned_text
            cell.font = self.bold_font

    def _add_second_line_template(self):
        """Add second line fix template"""
        rows = [
            [
                "#",
                "Object/Attribute Name",
                "Class ID",
                "Ver.",
                "OBIS Object Code / Default Value",
                "A.1",
                "A.2",
                "A.3",
                "A.4",
            ]
        ]
        self._append_rows(rows)
        for cell in self.sheet["2:2"]:
            cell.fill = self.light_gray_bg_color
            cell.alignment = self.center_aligned_text

    def add_energy_profile_daily_snapshot(self, id_class, id_attr):
        # dodamo rdečo vrstico
        rows = [["", "Energy Profile (Daily snapshot)"]]
        self._append_rows(rows)
        current_row = self.sheet._current_row
        for cell in self.sheet[f"{current_row}:{current_row}"]:
            cell.fill = self.red_bg_color
            cell.font = self.bold_font

        # dodamo rumeno vrstico
        calss_id, version = self._get_class_id_version(id_attr)
        obis_code = self._get_obis_code_daily(id_class)

        rows = [
            ["Attr", "Energy Profile (Daily snapshot)", calss_id, version, obis_code]
        ]
        self._append_rows(rows)
        current_row = self.sheet._current_row
        for cell in self.sheet[f"{current_row}:{current_row}"]:
            cell.fill = self.yellow_bg_color
            cell.font = self.bold_font

        # dodamo atribute
        table_len = self._add_atributes_table(id_attr)
        current_row = self.sheet._current_row
        for a in self.sheet[f"A{current_row-table_len+1}:B{current_row}"]:
            for cell in a:
                cell.fill = self.cyan_bg_color

        # dodamo prazno vrstico
        self.sheet._current_row += 1

    def add_total_energy_registers(self, id_class, id_attr):
        # dodamo rdečo vrstico
        rows = [["", "Total Energy Registers"]]
        self._append_rows(rows)
        current_row = self.sheet._current_row
        for cell in self.sheet[f"{current_row}:{current_row}"]:
            cell.fill = self.red_bg_color
            cell.font = self.bold_font

        # dodamo rumeno vrstico
        calss_id, version = self._get_class_id_version(id_attr)
        obis_code = self.tables[0][1][-1].strip()

        rows = [["Attr", "Active energy import (+A)", calss_id, version, obis_code]]
        self._append_rows(rows)
        current_row = self.sheet._current_row
        for cell in self.sheet[f"{current_row}:{current_row}"]:
            cell.fill = self.yellow_bg_color
            cell.font = self.bold_font

        # dodamo atribute
        table_len = self._add_atributes_table(id_attr)
        current_row = self.sheet._current_row
        for a in self.sheet[f"A{current_row-table_len+1}:B{current_row}"]:
            for cell in a:
                cell.fill = self.cyan_bg_color

        # dodamo prazno vrstico
        self.sheet._current_row += 1

        # dodamo rumeno vrstico
        calss_id, version = self._get_class_id_version(id_attr)
        obis_code = self.tables[0][2][-1].strip()

        rows = [["Attr", "Active energy import (-A)", calss_id, version, obis_code]]
        self._append_rows(rows)
        current_row = self.sheet._current_row
        for cell in self.sheet[f"{current_row}:{current_row}"]:
            cell.fill = self.yellow_bg_color
            cell.font = self.bold_font

        # dodamo atribute
        table_len = self._add_atributes_table(id_attr)
        current_row = self.sheet._current_row
        for a in self.sheet[f"A{current_row-table_len+1}:B{current_row}"]:
            for cell in a:
                cell.fill = self.cyan_bg_color

        # dodamo prazno vrstico
        self.sheet._current_row += 1

    def save_table(self, table_path):
        self.workbook.save(filename=table_path)

if __name__ == "__main__":
    my_parser = DocxTableReportParser(
        "Dodatni_primeri/data/PRIMER_kompleksna_tabela.docx"
    )

    my_parser.add_total_energy_registers(0, 1)
    my_parser.add_energy_profile_daily_snapshot(2, 3)
    my_parser.save_table("data/complex_report.xlsx")


__copyright__ = "Copyright (c) 2020 Jina AI Limited. All rights reserved."
__license__ = "Apache-2.0"

import numpy as np

from .. import BaseAudioEncoder
from ...decorators import batching, as_ndarray

class MFCCTimbreEncoder(BaseAudioEncoder):
    """
    :class:`MFCCTimbreEncoder` is based on Mel-Frequency Cepstral Coefficients (MFCCs) which represent timbral features.
    :class:`MFCCTimbreEncoder` encodes an audio signal from a `Batch x Signal Length` ndarray into a
    `Batch x Concatenated Features` ndarray.
    """

    def __init__(self, input_sample_rate: int = 22050, n_mfcc: int = 20, n_fft_length: int = 2048,
                 hop_length: int = 512, *args, **kwargs):
        """
        :class:`MFCCTimbreEncoder` extracts from an audio signal a `n_mfcc`-dimensional feature vector for each MFCC
        frame.

        :param input_sample_rate: input sampling rate in Hz (22050 by default)
        :param n_mfcc: the number of coefficients (20 by default)
        :param n_fft: length of the FFT window (2048 by default)
        :param hop_length: the number of samples between successive MFCC frames (512 by default)
        """
        super().__init__(*args, **kwargs)
        self.input_sample_rate = input_sample_rate
        self.n_mfcc = n_mfcc
        self.n_fft_length = n_fft_length
        self.hop_length = hop_length

    @batching
    @as_ndarray
    def encode(self, data: np.ndarray, *args, **kwargs) -> np.ndarray:
        """
        Segments the audio signal of each Chunk into short MFCC frames, extracts MFCCs for each frame and concatenates
        Chunk frame MFCCs into a single Chunk embedding.

        :param data: a `Batch x Signal Length` ndarray, where `Signal Length` is a number of samples
        :return: a `Batch x Concatenated Features` ndarray, where `Concatinated Features` is a `n_mfcc`-dimensional
        feature vector times the number of the MFCC frames
        """
        from librosa.feature import mfcc
        embeds = []
        for chunk_data in data:
            mfccs = mfcc(y=chunk_data, sr=self.input_sample_rate, n_mfcc=self.n_mfcc, n_fft=self.n_fft_length,
                         hop_length=self.hop_length)
            embeds.append(mfccs.flatten())
        return embeds

class ChromaPitchEncoder(BaseAudioEncoder):
    """
    :class:`ChromaPitchEncoder` is based on chroma spectrograms (chromagrams) which represent melodic/harmonic features.
    :class:`ChromaPitchEncoder` encodes an audio signal from a `Batch x Signal Length` ndarray into a
    `Batch x Concatenated Features` ndarray.
    """

    def __init__(self, input_sample_rate: int = 22050, hop_length: int = 512, *args, **kwargs):
        """
        :class:`ChromaPitchEncoder` extracts from an audio signal a 12-dimensional feature vector, that refer to 12
        octaves, for each chroma frame.

        :param input_sample_rate: input sampling rate in Hz (22050 by default)
        :param hop_length: the number of samples between successive chroma frames (512 by default)
        """
        super().__init__(*args, **kwargs)
        self.input_sample_rate = input_sample_rate
        self.hop_length = hop_length

    @batching
    @as_ndarray
    def encode(self, data: np.ndarray, *args, **kwargs) -> np.ndarray:
        """
        Segments the audio signal of each Chunk into short chroma frames, extracts chromagrams for each frame and
        concatenates Chunk frame chromagrams into a single Chunk embedding.

        :param data: a `Batch x Signal Length` ndarray, where `Signal Length` is a number of samples
        :return: a `Batch x Concatenated Features` ndarray, where `Concatinated Features` is a 12-dimensional feature
        vector times the number of the chroma frames
        """
        from librosa.feature import chroma_cqt
        embeds = []
        for chunk_data in data:
            chromagrams = chroma_cqt(y=chunk_data, sr=self.input_sample_rate, n_chroma=12, hop_length=self.hop_length)
            embeds.append(chromagrams.flatten())
        return embeds


#!/usr/bin/python3
# coding: utf-8

import os, sys, stat
import random
import threading
import multiprocessing
import subprocess
import streams.transcribe
import queue
import select
import time,datetime
from fcntl import fcntl, F_GETFL, F_SETFL, ioctl
import cv2
import numpy as np
import re
import select
import setproctitle

from streams.detectors import shot

# TODO:
#  structure this more intelligently
def new_frame(frame_num=0):
  frame = {}
  frame['frame'] = frame_num
  # TODO: initialize this properly
  frame['frametime'] = ''

  # TODO: initialize these properly
  #frame['small'] = bytes()
  #frame['gray'] = bytes()
  #frame['rframe'] = bytes()

  # dictates what kind of shot boundary detected
  frame['shot_type'] = ''
  # edge contours, good for rough estimations
  frame['edges'] = []
  # proprietary metric
  frame['gray_motion_mean'] = 0
  # another video break metric
  frame['mse'] = 0
  # shot break detection score
  frame['sbd'] = 0.0
  # hulls from motion
  frame['motion_hulls'] = []
  frame['viewport'] = None

  # last shot detected, could be itself
  frame['shot_detect'] = 0
  # last motion frame detected, could be itself
  frame['motion_detect'] = 0
  # last audio break detected, could be itself
  frame['audio_detect'] = 0
  # last scene break detected, could be itself
  frame['scene_detect'] = 0
  # type of audio detected
  frame['audio_type'] = ''
  # how long has this frame been breaking?
  frame['break_count'] = 0
  # zeros in the audio frame - a measure of artificial silence
  frame['zeros'] = 0
  frame['zcr'] = 0
  frame['zcrr'] = 0
  frame['ste'] = 0
  frame['ster'] = 0

  # is this a super quiet scene or not?
  frame['isolation'] = False

  frame['raudio_max'] = 100
  frame['raudio_mean'] = 100
  frame['audio_max'] = 100
  frame['audio_mean'] = 100
  frame['abd_thresh'] = 0
  frame['ster_thresh'] = 0

  # how much speech is in the frame
  frame['speech_level'] = 0.0

  frame['ssd_rects'] = []
  frame['human_rects'] = []
  frame['face_rects'] = []
  frame['vehicle_rects'] = []
  frame['object_rects'] = []
  frame['contrib_rects'] = []
  frame['text_rects'] = []
  frame['text_hulls'] = []

  frame['com_detect'] = None

  return frame

#
# given a segment of audio, determine its energy
#  check if that energy is in the human band of
#  speech.
#
#  warning - this has a defect in it that will crash for long
#  segments of audio.  Needs to be fixed
#
def speech_calc(self,raw_audio):
  if len(raw_audio) < 10:
    return 0

  data_freq = np.fft.fftfreq(len(raw_audio),1.0/self.sample)
  data_freq = data_freq[1:]
  data_ampl = np.abs(np.fft.fft(raw_audio))
  data_ampl = data_ampl[1:]
  data_energy = data_ampl ** 2

  energy_freq = {}
  for (i, freq) in enumerate(data_freq):
    if abs(freq) not in energy_freq:
      energy_freq[abs(freq)] = data_energy[i] * 2
  sum_energy = 0
  for f in energy_freq.keys():
    if 300<f<3000:
      sum_energy += energy_freq[f]
  if sum_energy < 1:
    return 0

  return sum_energy / sum(energy_freq.values())

#
# This is the main thread that processes ffmpeg,
# and muxes everything together out of the fifos
#
class FileVideoStream:

  def process(self):

    if self.image:
      return

      #
      # self.pipe is the pipe from the ffmpeg subprocess
      # self.video_fifo is the video channel from the ffmpeg proces
      # self.audio_fifo is the audio channel from the ffmpeg proces
      # self.caption_fifo is the caption channel from the ffmpeg proces
      #   captions are pulled from a filter as they are embedded in the
      #   video meta data and otherwise lost on the transcode
      #
    if self.pipe is not None:
      self.pipe.terminate()
    if self.video_fifo is not None:
      os.close(self.video_fifo)
    if self.audio_fifo is not None:
      os.close(audio_fifo)
    if self.caption_fifo is not None:
      os.close(caption_fifo)

    if os.path.lexists('/tmp/%s_video' % self.name):
      os.unlink('/tmp/%s_video' % self.name)
    if os.path.lexists('/tmp/%s_audio' % self.name):
      os.unlink('/tmp/%s_audio' % self.name)
    if os.path.lexists('/tmp/%s_caption' % self.name):
      os.unlink('/tmp/%s_caption' % self.name)

    os.mkfifo('/tmp/%s_video' % self.name)
    os.mkfifo('/tmp/%s_audio' % self.name)
    os.mkfifo('/tmp/%s_caption' % self.name)

    video_command = []
    #
    #  big assumption: http stream is an MPEGTS and probably a TV source
    # TODO: ffmpeg command passthrough.  enable / disable logging
    #
    if self.stream[:4] != 'rtsp':
      # probably should do this better
      self.stream = str.replace(self.stream,':','\\\:')
      self.stream = str.replace(self.stream,'@','\\\@')

      video_command = [ '-hide_banner','-loglevel','panic','-hwaccel','vdpau','-y','-f','lavfi','-i','movie=%s:s=0\\\:v+1\\\:a[out0+subcc][out1]' % self.stream,'-map','0:v','-vf','bwdif=0:-1:1,scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2' % (self.width,self.height,self.width,self.height),'-pix_fmt','bgr24','-r','%f' % self.fps,'-s','%dx%d' % (self.width,self.height),'-vcodec','rawvideo','-f','rawvideo','/tmp/%s_video' % self.name, '-map','0:a','-acodec','pcm_s16le','-ac','1','-ar','%d' % self.sample,'-f','wav','/tmp/%s_audio' % self.name, '-map','0:s','-f','srt','/tmp/%s_caption' % self.name  ]
      if self.seek:
        video_command[:0] = ['-ss',self.seek]
      video_command[:0] = ['ffmpeg']
      print('ffmpeg cmd',' '.join(video_command))
      self.pipe = subprocess.Popen(video_command)

      print('Step 2 initializing video /tmp/%s_video' % self.name)
      self.video_fifo = os.open('/tmp/%s_video' % self.name,os.O_RDONLY | os.O_NONBLOCK,self.width*self.height*3*10)
      #
      # WARNING: your fifo pipe has to be large enough to hold a bunch of video frames.  By default its only 1MB, which is
      #  not very much data.  I increased mine to hold 10 frames of 1080 rawvideo
      #
      # sudo sysctl fs.pipe-max-size=6220800
      fcntl(self.video_fifo,1031,6220800)
      #
      # puny linux default
      #fcntl(self.video_fifo,1031,1048576)

      print('Step 3 initializing audio /tmp/%s_audio' % self.name)
      self.audio_fifo = os.open('/tmp/%s_audio' % self.name,os.O_RDONLY | os.O_NONBLOCK,self.sample*2*10)
      #fcntl(self.audio_fifo,1031,1048576)
      fcntl(self.audio_fifo,1031,6220800)

      # TODO: move this to os.open()
      self.caption_fifo = os.open('/tmp/%s_caption' % self.name,os.O_RDONLY | os.O_NONBLOCK,4096)
    #
    # big assumption rtsp stream is h264 of some kind and probably a security camera
    #  it does not handle audio, as the audio not actually in the stream.  Presumably we could mux it here if it exists
    elif self.stream[:4] == 'rtsp':
      # we don't need the lavfi command because there are no subs.  also, the movie= container for rtsp doesn't let me pass rtsp_transport, which will result in
      #  dropped packets if I do not do
      # also, vdpau is not working for some reason (yuv issues)
      #
      video_command = [ 'ffmpeg','-nostdin','-re','-hide_banner','-loglevel','panic','-y','-r','%f' % self.fps,'-rtsp_transport','tcp','-i',self.stream,'-map','0:v','-vf','scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2' % (self.width,self.height,self.width,self.height),'-pix_fmt','bgr24','-r','%f' % self.fps,'-vcodec','rawvideo','-f','rawvideo','/tmp/%s_video' % self.name ]

      self.pipe = subprocess.Popen(video_command)
      print('fifo: /tmp/%s_video' %self.name)
      self.video_fifo = os.open('/tmp/%s_video' % self.name,os.O_RDONLY | os.O_NONBLOCK,self.width*self.height*3*10)
    else:
      print("unrecognized input")
      return

    self.audio_poll = None
    self.video_poll = None
    self.caption_poll = None
    if self.caption_fifo:
      self.caption_poll = select.poll()
      self.caption_poll.register(self.caption_fifo,select.POLLIN)
    if self.video_fifo:
      self.video_poll = select.poll()
      self.video_poll.register(self.video_fifo,select.POLLIN)
    if self.audio_fifo:
      self.audio_poll = select.poll()
      self.audio_poll.register(self.audio_fifo,select.POLLIN)
    print('ffmpeg pid:',self.pipe.pid)

#
# This is the queue that holds everything together
#
  def __init__(self, source, queueSize=2048):

    self.streams = []

    # TODO: if source has ;##:##:## then seek start time to there
    self.seek = None
    if source[-9] == ';':
      self.seek = source[-8:]
      source = source[0:-9]

    self.stream = source
    self.image = False
    if self.stream.endswith('.jpg') or self.stream.endswith('.png'):
      self.image = True
    self.name = 'input_%d_%d' % (os.getpid(),random.choice(range(1,1000)))

    self.display = False
    if os.environ.get('DISPLAY'):
      self.display = True

    self.caption_guide = {}
    self.filename = None
    self.state = None
    self.pipe = None
    #self.transcribe = None
    self.video_fifo = None
    self.audio_fifo = None
    self.caption_fifo = None
    self.age = 0
    self.filepos = 0
    self.clockbase = 0
    self.microclockbase = 0
    self.stopped = multiprocessing.Value('i',0)
    self.ff = False

    # disabled for now.
    self.tas = None
    #self.tas = TranscribeAudioStream().load()

    # initialize the analyzer pipe
    self.Q = multiprocessing.Queue(maxsize=queueSize)

  def __del__(self):
    if self.pipe:
      # not cleanly exiting the threads leads to all kinds of problems, including deadlocks
      outs, errs = self.pipe.communicate(timeout=15)
      self.pipe.poll()
      self.pipe.wait()
      self.pipe.terminate()
      self.pipe.kill()

    if self.video_fifo:
      os.close(self.video_fifo)
    if self.audio_fifo:
      os.close(self.audio_fifo)
    if self.caption_fifo:
      os.close(self.audio_fifo)

    # I need to figure out how to get it to reset the console too
    if os.path.lexists('/tmp/%s_video' % self.name):
      os.unlink('/tmp/%s_video' % self.name)
    if os.path.lexists('/tmp/%s_audio' % self.name):
      os.unlink('/tmp/%s_audio' % self.name)
    if os.path.lexists('/tmp/%s_caption' % self.name):
      os.unlink('/tmp/%s_caption' % self.name)

  def load(self,width,height,fps,sample,scale=0.2):
    setproctitle.setproctitle('v2d main')
    self.scale = scale
    self.width = width
    self.height = height
    self.fps = fps
    self.sample = sample
    self.t = multiprocessing.Process(target=self.update, args=())
    self.t.daemon = True
    self.t.start()
    return self

# this is the demuxer.  it puts the video frames into
#  a queue.  the captions go into neverending hash (never gets pruned)
#  audio goes into a neverending place in memory
  def update(self):
     print("process!",self.stopped.value)
     setproctitle.setproctitle('v2d reader')
     self.process()
     timeout = 0
     read_frame = 0

     start_talking = 0
     no_talking = 0

     data_buf = []
     last_buf = None
     last_scene = 0

     rolling_ste = []

     last_rms = 100
     play_audio = None
     raw_audio = bytes()
     raw_image = bytes()

# TODO: get this into utils/var
     data = new_frame(read_frame)
     data['fps'] = self.fps
     while self.stopped.value == 0:
       bstart = time.time()

       # TODO: replace with qmax for both
       if self.Q.qsize() == 1023:
         print('running fast, sleeping')
       if self.Q.qsize() >= 100:
         time.sleep(0.01)
       if self.Q.qsize() >= self.fps*3:
         time.sleep(0.1)
         continue

       if self.audio_poll is None and self.video_poll is None and self.image is False:
         break

###### captions
#  4096 was kind of chosen at random.  Its a nonblocking read
       if read_frame % self.fps == 0 and self.caption_fifo and self.caption_poll is not None:
         # doesn't really need to do this on every frame
         events = self.caption_poll.poll(1)
         if len(events) > 0 and events[0][1] & select.POLLIN:
           raw_subs = os.read(self.caption_fifo,4096)
           if raw_subs and len(raw_subs) > 0:
             data['caption_time'],data['caption'] = self.parse_caption(raw_subs)

###### audio
#  audio is read in, a little bit at a time
       if self.audio_fifo and self.audio_poll is not None and data.get('audio') is None:
         fail = 0
         bufsize = int((self.sample*2/self.fps) - len(raw_audio))
         events = self.audio_poll.poll(1)
         #print('audio events',events)
         if len(events) > 0 and not events[0][1] & select.POLLIN:
           self.audio_poll.unregister(self.audio_fifo)
           self.audio_poll = None
           os.close(self.audio_fifo)
           self.audio_fifo = None
           print('warning audio error',events[0][1])
           continue

         while bufsize > 0 and len(events) > 0 and events[0][1] & select.POLLIN:
           tmp = os.read(self.audio_fifo,bufsize)
           events = self.audio_poll.poll(1)
           if tmp is not None:
             raw_audio += tmp
             bufsize = int((2 * self.sample / self.fps) - len(raw_audio))
             if bufsize < 0:
               print('warning, negative buf',bufsize)
           else:
             fail += 1
             print("warning audio underrun0",len(raw_audio))
             time.sleep(0.001)
           if fail > 5:
             print("warning audio underrun1",len(raw_audio))
             break
         if raw_audio is not None and len(raw_audio) == int(self.sample*2/self.fps):
           data['audio'] = raw_audio
           raw_audio = bytes()

       if data is not None and data.get('audio') is not None and data.get('audio_np') is None:
         data['audio_np'] = np.fromstring(data['audio'],np.int16)

         if play_audio is None:
           play_audio = data['audio_np'].copy()
         else:
           play_audio = np.concatenate((play_audio,data['audio_np']),axis=0)
         if(len(play_audio) > self.sample):
           play_audio = play_audio[int(self.sample / self.fps):]

         # speed up fft by taking a sample
         mags2 = np.fft.rfft(abs(play_audio[0::100]))
         mags = np.abs(mags2)
         amax = max(mags)
         amin = min(mags)
         variance = np.var(mags)
         data['last_abd'] = (amax + amin) / variance
         data['last_audio'] = play_audio
         data['last_audio_level'] = np.std(abs(play_audio),axis=0)
         data['last_audio_power'] = np.sum(abs(play_audio)) / self.sample
         local_max2 = np.max(play_audio)
         local_max = np.max(play_audio**2)
         local_mean = np.mean(play_audio**2)
         #print('max',local_max2,local_mean)
         # TODO: fix issues with negative sqrts
         data['last_rms'] = np.sqrt(local_mean)
         data['last_audio_max'] = abs(play_audio).max(axis=0)
         data['last_audio_mean'] = abs(play_audio).mean(axis=0)
         last = data['last_audio'][0::1000]
         # WARNING: compute intensive
         data['last_ste'] = sum( [ abs(x)**2 for x in last ] ) / len(last)
         #print('last_',data['frame'],data['last_ste'])
         #print('last_',data['last_audio_mean'],play_audio_mean,delta)

         # TODO: misnamed
         data['audio_level'] = np.std(abs(data['audio_np']),axis=0)
         data['audio_power'] = np.sum(abs(data['audio_np'])) / self.sample
         local_mean = np.mean(data['audio_np']**2)
         # TODO: fix issues with negative sqrts
         if data['audio_level'] > 0 and local_mean > 0:
           data['rms'] = np.sqrt(local_mean)
         data['audio_max'] = abs(data['audio_np']).max(axis=0)
         data['audio_mean'] = abs(data['audio_np']).mean(axis=0)

         # WARNING: compute intensive
         data['ste'] = sum( [ abs(x)**2 for x in data['audio_np'][0::100] ] ) / len(data['audio_np'][0:100])
         mags2 = np.fft.rfft(abs(data['audio_np']))
         mags = np.abs(mags2)
         amax = max(mags)
         amin = min(mags)
         variance = np.var(mags)
         data['abd'] = (amax + amin) / (variance + 0.0001)

         signs = np.sign(data['audio_np'])
         signs[signs == 0] = -1
         data['zcr'] = len(np.where(np.diff(signs))[0])/(len(data['audio_np'] + 0.0001))

         signs = np.sign(data['last_audio'])
         signs[signs == 0] = -1
         data['last_zcr'] = len(np.where(np.diff(signs))[0])/(len(data['last_audio'] + 0.0001))

         data['ster'] = data['last_ste'] / (data['ste'] + 0.0001)
# it might make sense to reset this on scene breaks
         rolling_ste.append(data['ster'])
         if len(rolling_ste) > self.fps * 10:
           rolling_ste.pop(0)
         data['mean_ste'] = np.mean(rolling_ste)
         data['pwrr'] = data['audio_power'] / (data['last_audio_power'] + 0.0001)
         data['zcrr'] = data['last_zcr'] / (data['zcr'] + 0.0001)

         # calculate longest continous zero chain
         longest = 0
         last = 0
         for i,e in enumerate(data['audio_np']):
           if e == 0:
             longest += 1
             if longest > last:
               last = longest
             else:
               longest = 0
         data['zeros'] = last

         data['speech_level'] = speech_calc(self,data['audio_np'])

         #print('last audio',read_frame)
         if data['speech_level'] > 0.8:
           #print("Talking!",data['speech_level'],read_frame)
           start_talking = read_frame
           no_talking = 0
         else:
           no_talking += 1
           if (no_talking == 10 and len(play_audio) > int(2*self.sample)):
             #print("  Transcribing:",len(play_audio),read_frame,read_frame - start_talking)
             if self.tas and self.tas.transcribe:
               self.tas.transcribe.stdin.write(play_audio)
             start_talking = 0

###### video
       if self.image:
         data['raw'] = cv2.resize(cv2.imread(self.stream),(self.width,self.height))
       elif self.video_fifo and self.video_poll is not None and data.get('raw') is None:
         fail = 0
         bufsize = self.width*self.height*3 - len(raw_image)
         events = self.video_poll.poll(1)
         if len(events) > 0 and not events[0][1] & select.POLLIN:
           self.video_poll.unregister(self.video_fifo)
           self.video_poll = None
           os.close(self.video_fifo)
           self.video_fifo = None
           print('warning video error',events[0][1])
           continue

         while bufsize > 0 and len(events) > 0 and events[0][1] & select.POLLIN:
           tmp = os.read(self.video_fifo,bufsize)
           events = self.video_poll.poll(1)
           if tmp is not None:
             raw_image += tmp
             bufsize = self.width*self.height*3 - len(raw_image)
           else:
             fail += 1
             print("warning video underrun0",len(raw_image))
             time.sleep(0.001)
           if fail > 5:
             print("warning video underrun1",len(raw_image))
             break
         if raw_image is not None and len(raw_image) == self.width*self.height*3:
           data['raw'] = np.fromstring(raw_image,dtype='uint8').reshape((self.height,self.width,3))
           raw_image = bytes()

       if data is not None and data.get('raw') is not None and data.get('rframe') is None:
         # crop out letter and pillarbox
         #  don't do this if we get a null  - its a scene break
         #  TODO: don't do this unless the height and width is changing by a lot
         data['small'] = cv2.resize(data['raw'],(int(data['raw'].shape[:2][1] * self.scale),int(data['raw'].shape[:2][0] * self.scale)))
         data['tiny'] = cv2.resize(data['raw'],(8,8))
         #non_empty_columns = np.int0(np.where(data['small'].max(axis=0) > 0)[0] / self.scale)
         # TODO: instead of resizing this, mark a flag
         #if len(non_empty_columns) > 100 and min(non_empty_columns) > self.height * 0.05 and max(non_empty_columns) < self.height * 0.95:
         #  data['rframe'] = data['raw'][0:self.height,min(non_empty_columns):max(non_empty_columns)+1:]
         #  data['small'] = cv2.resize(data['rframe'],(int(data['rframe'].shape[:2][1] * self.scale),int(data['rframe'].shape[:2][0] * self.scale)))
         #else:
         data['rframe'] = data['raw']

         data['height'], data['width'], data['channels'] = data['rframe'].shape
         data['scale'] = self.scale

         data['gray'] = cv2.cvtColor(data['small'],cv2.COLOR_BGR2GRAY)
         data['hsv'] = cv2.cvtColor(data['small'],cv2.COLOR_BGR2HSV)

         data['lum'] = np.sum(data['hsv']) / float(data['hsv'].shape[0] * data['hsv'].shape[1] * data['hsv'].shape[2])
         data['frame_mean'] = np.sum(data['small']) / float(data['small'].shape[0] * data['small'].shape[1] * data['small'].shape[2])
         data['hist'] = cv2.calcHist([data['small']], [0, 1, 2], None, [8, 8, 8],[0, 256, 0, 256, 0, 256])
         data['hist'] = cv2.normalize(data['hist'],5).flatten()
         hist = data['hist']
         hist = hist.ravel()/hist.sum()
         logs = np.log2(hist+0.00001)
         data['contrast'] = -1 * (hist*logs).sum()
         #print('last video',read_frame)

         data['original'] = data['rframe'].copy()
         data['show'] = data['rframe'].copy()

###### transcription
       if self.tas and self.tas.transcribe and self.tas.transcribe.stdout:
         frame_transcribe = self.tas.transcribe.stdout.read(4096)
         if frame_transcribe is not None:
           data['transcribe'] = frame_transcribe.decode('ascii')
           print("  Transcribe:",len(play_audio),frame_transcribe)

       # drop frames if necessary, only if URL
       if self.audio_fifo and self.video_fifo:
         if data.get('audio_np') is not None and data.get('rframe') is not None:
           self.microclockbase += 1 / self.fps
           # instead of directly playing the video, buffer it a little
           # buffer this for 30 frames and send them all at the same time
           #  this way we can stitch the audio together
           window = self.fps

           # TODO: don't do this in the middle of a break, cycle around and do it again
           # this prevents the buffer from cutting in the middle of a break cluster
           #  NOTE: defect might be on the other side -- i.e., the :2 cuts the break in half
           if read_frame % (2*window) == 0 and read_frame > 0 and data['shot_detect'] > data['frame'] - 5:
             print('WARNING UPCOMING SPLIT SCENE, PLEASE FIX')
           if read_frame % (2*window) == 0 and read_frame > 0:
             cast_audio = bytes()
             for buf in data_buf[0:2*window]:
               cast_audio += buf['audio']
             data_buf[0]['play_audio'] = cast_audio

             # TODO: if no audio, pad with zero bytes
             # this is basically a NMS algorithm.  Find all the upcoming breaks
             #  and take the strongest one
             new_buf = []
             breaks = []
             scenes = []
             data_buf.append(data)
             for buf in data_buf:
               if last_buf is not None and buf['sbd'] == 0.0:
                 buf = shot.frame_to_contours(buf,last_buf)
                 buf = shot.frame_to_shots(buf,last_buf)
               if buf['scene_detect'] == buf['frame']:
                 scenes.append((buf['shot_detect'],buf['sbd']))
               if buf['shot_detect'] == buf['frame']:
                 breaks.append((buf['frame'],buf['sbd']))
               last_buf = buf
               new_buf.append(buf)

             real_break = new_buf[0]['shot_detect']
             # pick the oldest frame in a tie
             if len(scenes) > 0:
               scenes.sort(key=lambda x: x[0],reverse=True)
               scenes.sort(key=lambda x: x[1],reverse=True)
               #print(data['frame'],'upcoming scenes',scenes)
               # TODO: include the cluster in the frame packet for logging
               real_break = scenes[0][0]
               last_scene = real_break
               rolling_ste = []
               play_audio = None
             elif len(breaks) > 0:
               breaks.sort(key=lambda x: x[0],reverse=True)
               breaks.sort(key=lambda x: x[1],reverse=True)
               #print('upcoming breaks',breaks)
               real_break = breaks[0][0]

             for buf in new_buf[0:2*window]:
               # set this frame as the real scene for the block
               if buf['scene_detect'] != last_scene:
                 buf['scene_detect'] = last_scene
               # set this frame as the real shot for the block
               buf['shot_detect'] = real_break
               self.Q.put_nowait(buf)
             data_buf = new_buf[2*window:]
           else:
             data_buf.append(data)
           read_frame += 1
           data = new_frame(read_frame)
           data['last_scene'] = last_scene
       elif self.video_fifo or self.image:
         if data.get('rframe') is not None:
           self.microclockbase += 1 / self.fps
           self.Q.put_nowait(data)
           read_frame += 1
           data = new_frame(read_frame)
       data['fps'] = self.fps
       data['sample'] = self.sample

       time.sleep(0.0001)

     if self.video_fifo:
       os.close(self.video_fifo)
     if self.audio_fifo:
       os.close(self.audio_fifo)

  def read(self):
    return self.Q.get()

  def stop(self):
    print('stop!')
    self.stopped.value = 1
    for i in range(self.Q.qsize()):
      self.Q.get()
    self.Q = None

#
# Turn the output from the lavfi filter into a dict with timestamps
# TODO: move this wherever it goes
#
  def parse_caption(self,line):
    clean = line.rstrip().decode('ascii', errors='ignore')
    cleanr = re.compile('<.*?>')
    clean = re.sub(cleanr,'',clean)
    cleanr = re.compile('\{.an\d\}')
    clean = re.sub(cleanr,' ',clean)
    last_caption = ''
    for line in clean.splitlines():
      mo = re.search("^(\d\d:\d\d:\d\d),\d+? \-\-\> (\d\d:\d\d:\d\d),\d+",line)
      if mo and mo.group():
        last_caption = datetime.datetime.strftime(datetime.datetime.strptime(mo.group(1),"%H:%M:%S") + datetime.timedelta(seconds=self.clockbase),'%H:%M:%S')
        self.caption_guide[last_caption] = {}
        # add this to the internal frame clock
        self.caption_guide[last_caption]['start'] = last_caption
        self.caption_guide[last_caption]['stop'] = datetime.datetime.strftime(datetime.datetime.strptime(mo.group(2),"%H:%M:%S") + datetime.timedelta(seconds=self.clockbase),'%H:%M:%S')
        self.caption_guide[last_caption]['caption'] = ''
      elif last_caption:
        if line.isdigit():
          self.caption_guide[last_caption]['scene'] = int(line) - 1
        else:
          # TODO: remove duplicates
          self.caption_guide[last_caption]['caption'] += line + ' '
        # TODO: HACK. this belongs in the logger
        #print('\t\t[%s] CC: ' % last_caption,line)
    if last_caption is not '':
      return last_caption,self.caption_guide[last_caption]['caption']
    return last_caption,''


import torch, os, sys
from torch.utils.data import Dataset, DataLoader
from Utils.niiparsing import load_nii
import torch.nn.functional as F
from glob import glob
import numpy as np
import matplotlib.pyplot as plt

"""
Represents the entire brats dataset that is downloaded. Is manipulated by the
torch DataLoader to extract batches of data.
"""
class BratsDataset(Dataset):

    """
    Constructor that loads paths to all data.
    """
    def __init__(self):

        self.x = []
        self.y = []

        for i, filename in enumerate(glob("../organized_data/HGG/*flair.nii.gz")):

            if i == 5:
                break

            # sys.stdout.write("\rFile: %d" % (i + 1))
            # sys.stdout.flush()

            split = filename.split('flair')
            self.extract_features(filename, split[0] + 'seg' + split[1])

        self.x = np.vstack(self.x)
        print(self.x.shape)
        self.x = np.array(self.x).reshape(-1, 6)
        self.y = np.array(self.y).reshape(-1, 1)

    """
    Uses 3x3x3 sliding window and 0 padding to go through brain images and extract features using all 27
    pixels. These features will be the features for only the central pixel.
    """
    def extract_features(self, x_path, y_path):

        # Loading and normalizing whole brain images
        x_arr = load_nii(x_path)
        x_arr = x_arr / x_arr.max()
        y_arr = load_nii(y_path)

        windows = []

        num_c = 0
        num_nc = 0

        for depth in range(0, x_arr.shape[2] - 2, 3):

            # Parameterizing sliding window
            for row in range(0, x_arr.shape[0], 3):
                for col in range(0, x_arr.shape[1], 3):

                    if x_arr[row + 1, col + 1, depth + 1] == 0:
                        num_c += 1
                        continue
                    else:
                        num_nc += 1

                    # Creating sliding window across slice
                    x_window = x_arr[row : row + 3, col : col + 3, depth : depth + 3]
                    y_window = y_arr[row + 1,
                                     col + 1,
                                     depth + 1]

                    # Flattening window
                    windows.append(np.array(x_window).reshape(-1))

                    # Extracting label for central pixel
                    if y_window == 0:
                        self.y.append([0])
                    else:
                        self.y.append([1])

        windows = np.array(windows).reshape(-1, 27)

        inp = np.concatenate((
                             np.mean(windows, axis=1).reshape(-1, 1),
                             np.std(windows, axis=1).reshape(-1, 1),
                             np.var(windows, axis=1).reshape(-1, 1),
                             np.max(windows, axis=1).reshape(-1, 1),
                             np.min(windows, axis=1).reshape(-1, 1),
                             np.median(windows, axis=1).reshape(-1, 1)), axis=1)

        self.x.append(inp)

    """
    Number of samples that have been downloaded.
    """
    def __len__(self):

        return len(self.x)

    """
    Gets a single sample from the dataset. Returns dictionary of metadata about the
    sample for later processing. Image itself is returned as a tensor.
    """
    def __getitem__(self, idx):

        if torch.is_tensor(idx):
            idx = idx.tolist()

        return {

            'x' : self.x[idx],
            'y' : self.y[idx]

        }

"""
Creates a torch DataLoader that loads data into memory more efficiently and handles batching.
"""
def get_dataloader(batch_size=64, shuffle=True, num_workers=4):

    dataset = BratsDataset()
    print("Dataset has been created")
    return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)


import os
import time
import cProfile
import pstats
import io
import inspect

from euphonic.cli.powder_map import main as powder_map
from utils import get_fc_path, get_create_results_path, get_fc_info, NTHREADS

def main(use_brille=False):
    fc_info = get_fc_info()
    fc = fc_info[3] # 3 = AmSulf
    fc_file = str(get_fc_path(fc['filename']))

    args = [fc_file,
            '--n-threads', str(NTHREADS),
            '-s', 'tmp.png',
            '--npts-density', '10000',
            '--dipole-parameter', fc['sdipole_parameter'],
            '-w', 'coherent',
            '--grid', '6', '6', '6',
            '--temperature', '5']

    if use_brille:
        args += ['--use-brille', '--brille-npts', fc['sbrille_npts']]

    powder_map(args)

if __name__ == '__main__':
    use_brille_opts = [False, True]
    out_file = f'{get_create_results_path()}/powder_map_cprofile_{int(time.time())}.txt'
    for use_brille in use_brille_opts:
        pr = cProfile.Profile()
        pr.enable()
        main(use_brille=use_brille)
        pr.disable()

        strio = io.StringIO()
        stats = pstats.Stats(pr, stream=strio).sort_stats('cumtime')
        stats.print_stats()

        with open(out_file, 'a') as f:
            f.write(f'use_brille={use_brille}\n')
            f.write(strio.getvalue())

    with open(out_file, 'a') as f:
        f.write(inspect.getsource(inspect.getmodule(inspect.currentframe())))


from __future__ import print_function

import argparse
import dace
import math
import numpy as np
import scipy
import pdb
import select
import sys

W = dace.symbol('W')
H = dace.symbol('H')
nnz = dace.symbol('nnz')
itype = dace.dtypes.uint32
dtype = dace.dtypes.float32

def make_pre_state(sdfg):

    state = sdfg.add_state("pre_state")

    a_row_host = state.add_array("A_row", (H + 1, ), itype)
    a_row_device = state.add_array("A_row_device", (H + 1, ),
                                   itype,
                                   transient=True,
                                   storage=dace.dtypes.StorageType.FPGA_Global)

    a_col_host = state.add_array("A_col", (nnz, ), itype)
    a_col_device = state.add_array("A_col_device", (nnz, ),
                                   itype,
                                   transient=True,
                                   storage=dace.dtypes.StorageType.FPGA_Global)

    a_val_host = state.add_array("A_val", (nnz, ), dtype)
    a_val_device = state.add_array("A_val_device", (nnz, ),
                                   dtype,
                                   transient=True,
                                   storage=dace.dtypes.StorageType.FPGA_Global)

    x_host = state.add_array("x", (W, ), dtype)
    x_device = state.add_array("x_device", (W, ),
                               dtype,
                               transient=True,
                               storage=dace.dtypes.StorageType.FPGA_Global)

    state.add_memlet_path(a_row_host,
                          a_row_device,
                          memlet=dace.memlet.Memlet.simple(
                              a_row_device, "0:H+1"))

    state.add_memlet_path(a_col_host,
                          a_col_device,
                          memlet=dace.memlet.Memlet.simple(
                              a_col_device, "0:nnz"))

    state.add_memlet_path(a_val_host,
                          a_val_device,
                          memlet=dace.memlet.Memlet.simple(
                              a_val_device, "0:nnz"))

    state.add_memlet_path(x_host,
                          x_device,
                          memlet=dace.memlet.Memlet.simple(x_device, "0:W"))

    return state

def make_post_state(sdfg):

    state = sdfg.add_state("post_state")

    b_device = state.add_array("b_device", (H, ),
                               dtype,
                               transient=True,
                               storage=dace.dtypes.StorageType.FPGA_Global)
    b_host = state.add_array("b", (H, ), dtype)

    state.add_memlet_path(b_device,
                          b_host,
                          memlet=dace.memlet.Memlet.simple(b_host, "0:H"))

    return state

def make_nested_sdfg(parent):

    sdfg = dace.SDFG("spmv_inner")

    set_zero_state = sdfg.add_state("set_zero")
    set_zero_b = set_zero_state.add_scalar(
        "b_buffer",
        dtype,
        transient=True,
        storage=dace.dtypes.StorageType.FPGA_Registers)
    set_zero_tasklet = set_zero_state.add_tasklet("set_zero", {}, {"b_out"},
                                                  "b_out = 0")
    set_zero_state.add_memlet_path(set_zero_tasklet,
                                   set_zero_b,
                                   src_conn="b_out",
                                   memlet=dace.memlet.Memlet.simple(
                                       set_zero_b, "0"))

    write_back_state = sdfg.add_state("write_back")
    write_back_b_buffer = write_back_state.add_scalar(
        "b_buffer",
        dtype,
        transient=True,
        storage=dace.dtypes.StorageType.FPGA_Registers)
    write_back_b = write_back_state.add_scalar(
        "b_write", dtype, storage=dace.dtypes.StorageType.FPGA_Registers)
    write_back_state.add_memlet_path(write_back_b_buffer,
                                     write_back_b,
                                     memlet=dace.memlet.Memlet.simple(
                                         write_back_b, "0"))

    state = sdfg.add_state("compute_cols")

    sdfg.add_edge(set_zero_state, state, dace.sdfg.InterstateEdge())
    sdfg.add_edge(state, write_back_state, dace.sdfg.InterstateEdge())

    compute_entry, compute_exit = state.add_map("compute_col",
                                                {"j": "rowptr:rowend"})

    indirection_tasklet = state.add_tasklet("indirection",
                                            {"x_val_in", "col_index"},
                                            {"lookup"},
                                            "lookup = x_val_in[col_index]")

    x_in = state.add_scalar("x_in",
                            dtype,
                            storage=dace.dtypes.StorageType.FPGA_Registers,
                            transient=True)

    compute_tasklet = state.add_tasklet("compute", {"a", "x_val_in"}, {"out"},
                                        "out = a * x_val_in")

    b_buffer = state.add_scalar("b_buffer",
                                dtype,
                                transient=True,
                                storage=dace.dtypes.StorageType.FPGA_Registers)
    rowptr = state.add_scalar("row_begin",
                              itype,
                              storage=dace.dtypes.StorageType.FPGA_Registers)
    rowend = state.add_scalar("row_end",
                              itype,
                              storage=dace.dtypes.StorageType.FPGA_Registers)
    a_val = state.add_array("A_val_read", (nnz, ),
                            dtype,
                            storage=dace.dtypes.StorageType.FPGA_Global)
    a_col = state.add_array("A_col_read", (nnz, ),
                            itype,
                            storage=dace.dtypes.StorageType.FPGA_Global)
    x = state.add_array("x_read", (W, ),
                        dtype,
                        storage=dace.dtypes.StorageType.FPGA_Global)

    compute_entry.add_in_connector("rowptr")
    state.add_memlet_path(rowptr,
                          compute_entry,
                          dst_conn="rowptr",
                          memlet=dace.memlet.Memlet.simple(rowptr, "0"))

    compute_entry.add_in_connector("rowend")
    state.add_memlet_path(rowend,
                          compute_entry,
                          dst_conn="rowend",
                          memlet=dace.memlet.Memlet.simple(rowend, "0"))

    state.add_memlet_path(a_val,
                          compute_entry,
                          compute_tasklet,
                          dst_conn="a",
                          memlet=dace.memlet.Memlet.simple(a_val, "j"))

    state.add_memlet_path(x,
                          compute_entry,
                          indirection_tasklet,
                          dst_conn="x_val_in",
                          memlet=dace.memlet.Memlet.simple(x, "0:W"))

    state.add_memlet_path(a_col,
                          compute_entry,
                          indirection_tasklet,
                          dst_conn="col_index",
                          memlet=dace.memlet.Memlet.simple(a_col, "j"))

    state.add_memlet_path(indirection_tasklet,
                          x_in,
                          src_conn="lookup",
                          memlet=dace.memlet.Memlet.simple(x_in, "0"))

    state.add_memlet_path(x_in,
                          compute_tasklet,
                          dst_conn="x_val_in",
                          memlet=dace.memlet.Memlet.simple(x_in, "0"))

    state.add_memlet_path(compute_tasklet,
                          compute_exit,
                          b_buffer,
                          src_conn="out",
                          memlet=dace.memlet.Memlet.simple(
                              b_buffer, "0", wcr_str="lambda a, b: a + b"))

    return sdfg

def make_main_state(sdfg):

    state = sdfg.add_state("spmv")

    a_row = state.add_array("A_row_device", (H + 1, ),
                            itype,
                            transient=True,
                            storage=dace.dtypes.StorageType.FPGA_Global)
    a_col = state.add_array("A_col_device", (nnz, ),
                            itype,
                            transient=True,
                            storage=dace.dtypes.StorageType.FPGA_Global)
    a_val = state.add_array("A_val_device", (nnz, ),
                            dtype,
                            transient=True,
                            storage=dace.dtypes.StorageType.FPGA_Global)
    x = state.add_array("x_device", (W, ),
                        dtype,
                        transient=True,
                        storage=dace.dtypes.StorageType.FPGA_Global)
    b = state.add_array("b_device", (H, ),
                        dtype,
                        transient=True,
                        storage=dace.dtypes.StorageType.FPGA_Global)

    row_entry, row_exit = state.add_map(
        "compute_row", {"i": "0:H"},
        schedule=dace.dtypes.ScheduleType.FPGA_Device)

    rowptr = state.add_scalar("rowptr",
                              itype,
                              storage=dace.dtypes.StorageType.FPGA_Registers,
                              transient=True)
    rowend = state.add_scalar("rowend",
                              itype,
                              storage=dace.dtypes.StorageType.FPGA_Registers,
                              transient=True)

    nested_sdfg = make_nested_sdfg(state)
    nested_sdfg_tasklet = state.add_nested_sdfg(
        nested_sdfg, sdfg,
        {"row_begin", "row_end", "A_val_read", "A_col_read", "x_read"},
        {"b_write"})

    state.add_memlet_path(a_row,
                          row_entry,
                          rowptr,
                          memlet=dace.memlet.Memlet.simple(
                              rowptr, "0", other_subset_str="i"))
    state.add_memlet_path(rowptr,
                          nested_sdfg_tasklet,
                          dst_conn="row_begin",
                          memlet=dace.memlet.Memlet.simple(rowptr, "0"))

    state.add_memlet_path(a_row,
                          row_entry,
                          rowend,
                          memlet=dace.memlet.Memlet.simple(
                              rowend, "0", other_subset_str="i + 1"))
    state.add_memlet_path(rowend,
                          nested_sdfg_tasklet,
                          dst_conn="row_end",
                          memlet=dace.memlet.Memlet.simple(rowend, "0"))

    state.add_memlet_path(a_val,
                          row_entry,
                          nested_sdfg_tasklet,
                          dst_conn="A_val_read",
                          memlet=dace.memlet.Memlet.simple(a_val, "0:nnz"))

    state.add_memlet_path(x,
                          row_entry,
                          nested_sdfg_tasklet,
                          dst_conn="x_read",
                          memlet=dace.memlet.Memlet.simple(x, "0:W"))

    state.add_memlet_path(a_col,
                          row_entry,
                          nested_sdfg_tasklet,
                          dst_conn="A_col_read",
                          memlet=dace.memlet.Memlet.simple(a_col, "0:nnz"))

    state.add_memlet_path(nested_sdfg_tasklet,
                          row_exit,
                          b,
                          src_conn="b_write",
                          memlet=dace.memlet.Memlet.simple(b, "i"))

    return state

def make_sdfg(specialize):

    if specialize:
        name = "spmv_fpga_{}x{}x{}".format(H.get(), W.get(), nnz.get())
    else:
        name = "spmv_fpga"
    sdfg = dace.SDFG(name)

    pre_state = make_pre_state(sdfg)
    main_state = make_main_state(sdfg)
    post_state = make_post_state(sdfg)

    sdfg.add_edge(pre_state, main_state, dace.sdfg.InterstateEdge())
    sdfg.add_edge(main_state, post_state, dace.sdfg.InterstateEdge())

    return sdfg

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("W", type=int)
    parser.add_argument("H", type=int)
    parser.add_argument("nnz", type=int)
    parser.add_argument("-specialize",
                        default=False,
                        action="store_true",
                        help="Fix all symbols at compile time/in hardware")
    args = vars(parser.parse_args())

    W.set(args["W"])
    H.set(args["H"])
    nnz.set(args["nnz"])

    print(
        'Sparse Matrix-Vector Multiplication {}x{} ({} non-zero elements, {}specialized)'
        .format(W.get(), H.get(), nnz.get(),
                "not " if not args["specialize"] else ""))

    A_row = dace.ndarray([H + 1], dtype=itype)
    A_col = dace.ndarray([nnz], dtype=itype)
    A_val = dace.ndarray([nnz], dtype=dtype)

    x = dace.ndarray([W], dtype)
    b = dace.ndarray([H], dtype)

    # Assuming uniform sparsity distribution across rows
    nnz_per_row = nnz.get() // H.get()
    nnz_last_row = nnz_per_row + (nnz.get() % H.get())
    if nnz_last_row > W.get():
        print('Too many nonzeros per row')
        exit(1)

    # RANDOMIZE SPARSE MATRIX
    A_row[0] = itype(0)
    A_row[1:H.get()] = itype(nnz_per_row)
    A_row[-1] = itype(nnz_last_row)
    A_row = np.cumsum(A_row, dtype=itype.type)

    # Fill column data
    for i in range(H.get() - 1):
        A_col[nnz_per_row*i:nnz_per_row*(i+1)] = \
            np.sort(np.random.choice(W.get(), nnz_per_row, replace=False))
    # Fill column data for last row
    A_col[nnz_per_row * (H.get() - 1):] = np.sort(
        np.random.choice(W.get(), nnz_last_row, replace=False))

    A_val[:] = np.random.rand(nnz.get()).astype(dtype.type)
    #########################

    x[:] = np.random.rand(W.get()).astype(dtype.type)
    #b[:] = dtype(0)

    # Setup regression
    A_sparse = scipy.sparse.csr_matrix((A_val, A_col, A_row),
                                       shape=(H.get(), W.get()))

    spmv = make_sdfg(args["specialize"])
    if args["specialize"]:
        spmv.specialize(dict(H=H, W=W, nnz=nnz))
    spmv(A_row=A_row, A_col=A_col, A_val=A_val, x=x, b=b, H=H, W=W, nnz=nnz)

    if dace.Config.get_bool('profiling'):
        dace.timethis('spmv', 'scipy', 0, A_sparse.dot, x)

    diff = np.linalg.norm(A_sparse.dot(x) - b) / float(H.get())
    print("Difference:", diff)
    if diff >= 1e-5:
        print("Validation failed.")
        print("Result:")
        print(b)
        print("Reference:")
        print(A_sparse.dot(x))
        print("Type \"debug\" to enter debugger, "
              "or any other string to quit (timeout in 10 seconds)")
        read, _, _ = select.select([sys.stdin], [], [], 10)
        if len(read) > 0 and sys.stdin.readline().strip().lower() == "debug":
            print("Entering debugger...")
            pdb.set_trace()
        else:
            print("Exiting...")
    print("==== Program end ====")
    exit(0 if diff <= 1e-5 else 1)


# pylint: disable=line-too-long
import tensorflow as tf
from tensorflow.contrib.rnn import (  # pylint: disable=E0611
    stack_bidirectional_dynamic_rnn,
)
from tsaplay.models.tsa_model import TsaModel
from tsaplay.utils.tf import (
    variable_len_batch_mean,
    attention_unit,
    lstm_cell,
    l2_regularized_loss,
    generate_attn_heatmap_summary,
    resolve_optimizer,
)
from tsaplay.utils.addons import addon, attn_heatmaps, early_stopping

class LcrRot(TsaModel):
    def set_params(self):
        return {
            # * From original paper
            "learning_rate": 0.1,
            "keep_prob": 0.5,
            "hidden_units": 300,
            "l2_weight": 1e-5,
            "optimizer": "Momentum",
            "momentum": 0.9,
            "initializer": tf.initializers.random_uniform(-0.1, 0.1),
            "bias_initializer": tf.initializers.zeros(),
            "lstm_initial_bias": 0,
            # ? Suggestions from https://github.com/NUSTM/ABSC/tree/master/models/ABSC_Zozoz
            "train_embeddings": False,
            "batch-size": 25,
            # "epochs": 50,
            "early_stopping_minimum_iter": 50,
            # ? Following approach of Moore et al. 2018, using early stopping
            "epochs": 300,
            "early_stopping_patience": 10,
            # "early_stopping_metric": "macro-f1",
            "early_stopping_metric": "loss",
        }

    @addon([attn_heatmaps, early_stopping])
    def model_fn(self, features, labels, mode, params):
        with tf.variable_scope("target_bi_lstm"):
            features["target_emb"] = tf.nn.dropout(
                features["target_emb"], keep_prob=params["keep_prob"]
            )
            target_hidden_states, _, _ = stack_bidirectional_dynamic_rnn(
                cells_fw=[lstm_cell(**params, mode=mode)],
                cells_bw=[lstm_cell(**params, mode=mode)],
                inputs=features["target_emb"],
                sequence_length=features["target_len"],
                dtype=tf.float32,
            )
            r_t = variable_len_batch_mean(
                input_tensor=target_hidden_states,
                seq_lengths=features["target_len"],
                op_name="target_avg_pooling",
            )

        with tf.variable_scope("left_bi_lstm"):
            features["left_emb"] = tf.nn.dropout(
                features["left_emb"], keep_prob=params["keep_prob"]
            )
            left_hidden_states, _, _ = stack_bidirectional_dynamic_rnn(
                cells_fw=[lstm_cell(**params, mode=mode)],
                cells_bw=[lstm_cell(**params, mode=mode)],
                inputs=features["left_emb"],
                sequence_length=features["left_len"],
                dtype=tf.float32,
            )

        with tf.variable_scope("right_bi_lstm"):
            features["right_emb"] = tf.nn.dropout(
                features["right_emb"], keep_prob=params["keep_prob"]
            )
            right_hidden_states, _, _ = stack_bidirectional_dynamic_rnn(
                cells_fw=[lstm_cell(**params, mode=mode)],
                cells_bw=[lstm_cell(**params, mode=mode)],
                inputs=features["right_emb"],
                sequence_length=features["right_len"],
                dtype=tf.float32,
            )

        with tf.variable_scope("left_t2c_attn"):
            left_hidden_states = tf.nn.dropout(
                left_hidden_states, keep_prob=params["keep_prob"]
            )
            r_l, left_attn_info = attention_unit(
                h_states=left_hidden_states,
                hidden_units=params["hidden_units"] * 2,
                seq_lengths=features["left_len"],
                attn_focus=r_t,
                init=params["initializer"],
                bias_init=params["bias_initializer"],
                sp_literal=features["left"],
            )

        with tf.variable_scope("right_t2c_attn"):
            right_hidden_states = tf.nn.dropout(
                right_hidden_states, keep_prob=params["keep_prob"]
            )
            r_r, right_attn_info = attention_unit(
                h_states=right_hidden_states,
                hidden_units=params["hidden_units"] * 2,
                seq_lengths=features["right_len"],
                attn_focus=r_t,
                init=params["initializer"],
                bias_init=params["bias_initializer"],
                sp_literal=features["right"],
            )

        target_hidden_states = tf.nn.dropout(
            target_hidden_states, keep_prob=params["keep_prob"]
        )

        with tf.variable_scope("left_c2t_attn"):
            r_t_l, left_target_attn_info = attention_unit(
                h_states=target_hidden_states,
                hidden_units=params["hidden_units"] * 2,
                seq_lengths=features["target_len"],
                attn_focus=tf.expand_dims(r_l, axis=1),
                init=params["initializer"],
                bias_init=params["bias_initializer"],
                sp_literal=features["target"],
            )

        with tf.variable_scope("right_c2t_attn"):
            r_t_r, right_target_attn_info = attention_unit(
                h_states=target_hidden_states,
                hidden_units=params["hidden_units"] * 2,
                seq_lengths=features["target_len"],
                attn_focus=tf.expand_dims(r_r, axis=1),
                init=params["initializer"],
                bias_init=params["bias_initializer"],
                sp_literal=features["target"],
            )

        generate_attn_heatmap_summary(
            left_attn_info,
            left_target_attn_info,
            right_target_attn_info,
            right_attn_info,
        )

        final_sentence_rep = tf.concat([r_l, r_t_l, r_t_r, r_r], axis=1)

        final_sentence_rep = tf.nn.dropout(
            final_sentence_rep, keep_prob=params["keep_prob"]
        )
        logits = tf.layers.dense(
            inputs=final_sentence_rep,
            units=params["_n_out_classes"],
            kernel_initializer=params["initializer"],
            bias_initializer=params["bias_initializer"],
        )

        loss = l2_regularized_loss(
            labels=labels, logits=logits, l2_weight=params["l2_weight"]
        )

        optimizer = resolve_optimizer(**params)

        return self.make_estimator_spec(
            mode=mode, logits=logits, optimizer=optimizer, loss=loss
        )


import paramiko

from src.server.Exceptions.FilesExceptionsSSH import FileCreationError
from src.server.commands.path_functions import find_file, list_files_recursive, find_directory

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from src.server.data.computer import Computer

from src.server.ssh.ssh_commands import does_path_exists_ssh, create_folder_ssh

def check_file_exists(computer: 'Computer', file_path: str) -> bool:
    """
    Checks if the file exists on the remote computer.
    :param computer: The computer to check on if the file exists.
    :param file_path: The path of the file to check.
    :return: True if the file exists, False otherwise.
    """
    ssh: paramiko.SSHClient = computer.ssh_session
    file_exists: bool = does_path_exists_ssh(ssh, file_path)

    if not file_exists:
        computer.log_error(f"The file path : '{file_path}' is not a valid path or does not exists.")

    return file_exists

def check_all_files_exists(computer: 'Computer') -> bool:
    """
    Checks if all the installation files exists on the remote computer.
    It compares all .py and .txt files in the client folder, with the files on the remote computer
    in the %USERPROFILE%/Infos.project_name folder
    :param computer: The computer to check on if the files exists.
    :return: True if all the files exists, False otherwise.
    """
    import os
    files: list[str] = [file for file in list_files_recursive(find_directory("client")) if file.endswith(".py")
                        or file.endswith(".txt")]
    files = [os.path.basename(file) for file in files]
    files = [os.path.join(computer.get_project_directory_on_client(), file) for file in files]

    for file in files:
        if not check_file_exists(computer, file):
            return False
    return True

def check_python_script_installed(computer: 'Computer') -> bool:
    """
    Checks if the python script is installed on the remote computer.
    :return: True if the script is installed, False otherwise.
    """
    ssh: paramiko.SSHClient = computer.ssh_session
    client_folder_path: str = computer.get_project_directory_on_client()
    install_exists: bool = does_path_exists_ssh(ssh, client_folder_path)

    if not install_exists:
        computer.log("There is not installation of the python script on the remote computer, because the folder is "
                     "not here.")
        return install_exists

    return check_all_files_exists(computer=computer)

def install_python_script(computer: 'Computer') -> bool:
    """
    Installs the update_windows python script on the remote computer. Overwrites the files if they already exist.
    :param computer: The computer to install the script on\n
    :return True if the upload was successful, False otherwise
    """
    import os

    ssh: paramiko.SSHClient = computer.ssh_session
    python_script_path: str = computer.get_project_directory_on_client()

    created = create_folder_ssh(computer, python_script_path)

    if not created:
        computer.log_error(f"Error while creating the folder {python_script_path}")
        raise FileCreationError(f"Error while creating the folder {python_script_path}")

    sftp = ssh.open_sftp()
    computer.log("Opened sftp communication to install the scripts and requirements")

    # TODO : Simplifier le code pour avoir les fichiers à upload
    files_to_upload: list[str] = [os.path.basename(file) for file in list_files_recursive(find_directory("client"))
                                  if not file.endswith(".log")]
    try:

        for file in files_to_upload:
            file_path = find_file(file)
            sftp.put(file_path, os.path.join(python_script_path, file))
            computer.log(f"File {file} uploaded to {python_script_path}")
    except (FileNotFoundError, FileExistsError, IOError):
        computer.log_error(f"Error while uploading the files {files_to_upload} to the remote computer.")
        return False

    sftp.close()
    return True

if __name__ == '__main__':
    import os


#!/usr/bin/env python3

import json
import argparse
from time import sleep
from enum import Enum
from pickle import Unpickler
from collections import defaultdict

from contexts import write_context_to_file
from wikidata import context_from_dump, is_not_deprecated
from wikidata import has_claims, has_qualifiers, maybe_entity_value
from wikidata import all_direct_instances_in_class, format_datavalue
from wikidata import all_direct_classes_for_values_of, has_meaningful_value

class Colouring(Enum):
    none = 1
    direction = 2
    qualifiers = 3
    classes = 4

PROPERTIES = []
COLOURINGS = {
    'none': Colouring.none,
    'direction': Colouring.direction,
    'qualifiers': Colouring.qualifiers,
    'classes': Colouring.classes,
    }

def colour_none(subject, prop, **kwargs):
    return {subject: {prop}}

def colour_direction(subject, prop, claim, **kwargs):
    results = colour_none(subject=subject, prop=prop, claim=claim, **kwargs)
    value = maybe_entity_value(claim)

    if value:
        results[value] = {'^{}'.format(prop)}

    return results

def colour_qualifiers(subject, prop, claim, labels, **kwargs):
    def _coloured(prop, pid, qualifier, reverse=False):
        return '{}{}@[{}:{}]'.format('^' if reverse else '',
                                     prop,
                                     pid,
                                     format_datavalue(qualifier, labels))

    if not has_qualifiers(claim):
        return colour_direction(subject=subject, prop=prop, claim=claim, labels=labels, **kwargs)

    results = {subject: set([])}
    value = maybe_entity_value(claim)
    if value:
        results[value] = set([])

    for pid, qualifiers in claim['qualifiers'].items():
        for qualifier in qualifiers:
            results[subject] |= {_coloured(prop, pid, qualifier)}
            if value:
                results[value] |= {_coloured(prop, pid, qualifier,
                                             reverse=True)}

    return results

def colour_classes(subject, prop, claim, labels, instances, **kwargs):
    value = maybe_entity_value(claim)

    if not value or not value in instances:
        return colour_direction(subject=subject, prop=prop, claim=claim,
                                labels=labels, instances=instances, **kwargs)

    results = {subject: set([]),
               value: set([]),
    }

    for qid in instances[value]:
        label = qid

        if qid in labels:
            label = '{} ({})'.format(labels[qid], qid)

        edge = '{}@<{}>'.format(prop, label)
        results[subject] |= {edge}
        if value:
            results[value] |= {'^{}'.format(edge)}

    return results

COLOURING_MAP = {
    Colouring.none: colour_none,
    Colouring.direction: colour_direction,
    Colouring.qualifiers: colour_qualifiers,
    Colouring.classes: colour_classes,
    }

def process_properties(labels,
                       instances,
                       subclasses,
                       properties=[],
                       colouring=Colouring.none,
                       filter_property=None,
                       filter_value=None,
                       filter_entities=None,
                       **kwargs):
    def process_entity(eid, entity):
        def _matches(pid):
            return all([not properties or pid in properties,
                        not filter_entities or pid in filter_entities,
            ])

        _colour = COLOURING_MAP[colouring]
        result = defaultdict(set)
        bg = {}

        if filter_entities is not None and eid not in filter_entities:
            return result, bg

        if filter_value is not None:
            if filter_property not in entity['claims']:
                return result, bg

            found = False
            for claim in entity['claims'][filter_property]:
                if is_not_deprecated(claim):
                    value = maybe_entity_value(claim)
                    if not value:
                        continue
                    elif value == filter_value:
                        found = True
                    elif (value in subclasses and
                          filter_value in subclasses[value]):
                        found = True
            if not found:
                return result, bg

        for prop, claims in entity['claims'].items():
            if _matches(prop):
                claimed = False
                for claim in claims:
                    if (is_not_deprecated(claim) and
                        has_meaningful_value(claim)):
                        coloured = _colour(subject=eid,
                                           prop=prop,
                                           claim=claim,
                                           labels=labels,
                                           instances=instances,
                                           subclasses=subclasses)

                        if filter_entities is not None:
                            value = maybe_entity_value(claim)
                            if value and value not in filter_entities:
                                continue

                        for ent in coloured:
                            result[ent] |= coloured[ent]

        return result, bg
    return process_entity

def postprocess(labels,
                instances,
                subclasses,
                properties=[],
                colouring=Colouring.none,
                filter_property=None,
                filter_value=None,
                filter_entities=None):
    def process_context(context, **kwargs):
        result = kwargs
        result['context'] = context
        result['labels'] = labels

        return result
    return process_context

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Generate a formal context '
                                     '(in Burmeister format) from a Wikidata '
                                     'JSON dump')
    parser.add_argument('dump',
                        help='path to Wikidata dump file')
    parser.add_argument('context',
                        help='path to output context file')
    parser.add_argument('--indexes',
                        required=True, dest='indexes',
                        help='path to helper indexes file')
    parser.add_argument('--property', '-p',
                        action='append', metavar='Pid',
                        dest='properties', default=[],
                        help='include property Pid in the context')
    parser.add_argument('--properties-in-class',
                        action='append', metavar='Qid',
                        dest='qids', default=[],
                        help='add direct instances of class Qid to context')
    parser.add_argument('--colouring',
                        choices=COLOURINGS.keys(), default='none',
                        help='use given colouring type')
    parser.add_argument('--item-filter-property',
                        metavar='Pid', dest='filter_property',
                        help='use property Pid as background knowledge')
    parser.add_argument('--item-filter-value',
                        metavar='Value', dest='filter_value',
                        help='use value Value as background knowledge')
    parser.add_argument('--language',
                        metavar='Lang', default='en',
                        help='include labels in language Lang')
    parser.add_argument('--entities-from-file',
                        metavar='Eidfile',
                        dest='eidfile', default=None,
                        help='restrict entities to those in Eidfile')

    args = parser.parse_args()
    properties = args.properties

    for qid in args.qids:
        properties += all_direct_instances_in_class(qid)

    colouring = COLOURINGS[args.colouring]
    kwargs = {'properties': properties,
              'colouring': COLOURINGS[args.colouring],
              'filter_property': args.filter_property,
              'filter_value': args.filter_value,
              }

    with open(args.indexes, 'rb') as idxfile:
        pickle = Unpickler(idxfile)
        indexes = pickle.load()
        kwargs.update(indexes)

    if args.eidfile is not None:
        entities = set([])
        with open(args.eidfile, 'r') as eidfile:
            for line in eidfile:
                entities |= {line.strip()}

        kwargs.update({'filter_entities': entities})

    process_entity = process_properties(**kwargs)
    result = context_from_dump(dump=args.dump,
                               properties_for_entity=process_entity,
                               postprocess=postprocess(**kwargs))

    with open(args.context, 'w') as outfile:
        write_context_to_file(outfile=outfile, **result)


import datetime
import dateutil.parser as dp

data = {}
rowCount = 24
colCount = 7

def getListOfEvents(data, targetText):
    listOfTargetEvents = []
    listOfOtherEvents = []
    for event in data["items"]:
        thisEventDesc = event.get('summary', '')
        if(targetText in thisEventDesc.lower()):
            listOfTargetEvents.append(event)
        else:
            listOfOtherEvents.append(event)
    return listOfTargetEvents, listOfOtherEvents

def getCalendarMatrix(listOfTargetEvents, listOfOtherEvents, targetText):
    dictOfTimeToEvent = {}

    targetEventTimes = [dp.parse(event["start"]["dateTime"])
                        for event in listOfTargetEvents if "start" in event and "dateTime" in event["start"]]
    otherEventTimes = [dp.parse(event["start"]["dateTime"])
                       for event in listOfOtherEvents if "start" in event and "dateTime" in event["start"]]
    combined = listOfTargetEvents + listOfOtherEvents
    combined = [x for x in combined if "start" in x and "dateTime" in x["start"]]
    for ind, event in enumerate(combined):
        dateTime = event["start"]["dateTime"]
        pyDT = dp.parse(dateTime)
        event["start"]["dateTime"] = pyDT
        dictOfTimeToEvent[pyDT] = event

    minTime = min(targetEventTimes)
    minTime = minTime.replace(hour=0, minute=0, second=0, microsecond=0)
    maxTime = max(targetEventTimes)
    diffTime = maxTime - minTime
    calMatrix = [[0 for x in range(diffTime.days + 1)]
                 for y in range(rowCount)]
    calMatrixOther = [[0 for x in range(diffTime.days + 1)]
                      for y in range(rowCount)]

    for time in sorted(targetEventTimes + otherEventTimes):
        day = time.day - minTime.day
        hour = time.hour
        if((day >= (diffTime.days + 1) or hour >= rowCount)):
            continue
        else:
            if(targetText in dictOfTimeToEvent[time].get("summary", '').lower()):
                calMatrix[hour][day] = 1
            else:
                calMatrixOther[hour][day] = 1

    return calMatrix, calMatrixOther, minTime.weekday(), minTime