# -*- coding: utf-8 -*-
"""
Created on Thu Feb  2 16:41:09 2023

@author: ugo.pelissier
"""

from matplotlib.image import imread
from tabulate import tabulate
from tabulate import tabulate
import matplotlib.pyplot as plt
import numpy as np
import os
import time
from scipy.linalg import interpolative, rq

#-----------------------------------------------------------------------------#
# RANDOMIZED SVD
#-----------------------------------------------------------------------------#
def rSVD(A,r,q,p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Target rank.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------
    Low-rank SVD approximation for A.

    """
    start = time.time()

    # Step 1 : Sample column space of A with random matrix
    m, n = A.shape
    l=r+p
    Omega = np.random.randn(n,l)
    Y = A @ Omega
    for k in range(q):
        Y = A @ (A.T @ Y)
    Q, R = np.linalg.qr(Y,mode='reduced')

    # Step 2: Compute SVD on projected Y = Q.T @ A
    Z = Q.T @ A
    UY, S, VT = np.linalg.svd(Z,full_matrices=0)
    U = Q @ UY

    end = time.time()

    return U, S, VT, (end-start)

def adaptative_randomized_range_finder(A, r, eps, max_iter):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Size of the standard gaussian vector used for error approximation.
    eps : float
        Tolerance for error approximation.
    max_iter : int
        Max number of iterations.

    Returns
    -------
    Q : numpy array
        Orthonormal matrix whose range approximates the range of X.
    approx_error_list : array of floats
        DESCRIPTION.
    true_error_list : array of floats
        DESCRIPTION.
    j : int
        Number of iterations.

    """
    m, n = A.shape
    omega = np.random.randn(n,r) # Gaussian vectors
    y = A @ omega

    # Stoping criteria
    threshold = eps/(10*np.sqrt(2/np.pi))
    approx_error = np.linalg.norm(y[:,-r:],axis=0).max()

    # Initialization
    Q = np.empty([m, 0])
    I = np.eye(m)
    approx_error_list = []
    true_error_list = []

    for j in range(max_iter):
        if (approx_error<threshold):
            # print("\nAdaptative randomized range finder converged in {} iterations.".format(j))
            return Q, np.array(approx_error_list), np.array(true_error_list), j

        else:

            # Project y onto Q
            y[:,j] = (I - Q @ Q.T) @ y[:, j]

            # Normalize and append
            q = y[:,j] / np.linalg.norm(y[:,j])
            Q = np.concatenate((Q, q.reshape(len(q),1)), axis=1)

            # New gaussian vector
            omega = np.random.randn(n,1)

            # Get its approximation error
            y_new = (I - Q @ Q.T) @ (A @ omega)

            # Append to y
            y = np.concatenate((y, y_new), axis=1)

            # Update previous y(i)
            for k in range(j+1,j+r):
                y[:,k] -= q*np.dot(q, y[:,k])

            # Compute new approx error
            approx_error = np.linalg.norm(y[:,-r:],axis=0).max()
            approx_error_list.append(approx_error)

            # True error (normally not computed)
            true_error = np.linalg.norm(((I - Q @ Q.T) @ A),ord=2)
            true_error_list.append(true_error)

    print("Maximum iteration reached. Consider increase max_iter.")

    return Q, np.array(approx_error_list), np.array(true_error_list), j

def adaptative_range_finder_rSVD(A, r, eps, max_iter, q, p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Size of the standard gaussian vector used for error approximation.
    eps : float
        Tolerance for error approximation.
    max_iter : int
        Max number of iterations.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------
    Low-rank SVD approximation for X.

    """
    start = time.time()

    # Step 1 : Sample column space of X with random P matrix
    Q, _, _, _ = adaptative_randomized_range_finder(A, r, eps, max_iter)

    # Step 2: Compute SVD on projected Y = Q.T @ A
    Y = Q.T @ A
    UY, S, VT = np.linalg.svd(Y,full_matrices=0)
    U = Q @ UY

    end = time.time()

    return U, S, VT, (end-start)

def randomized_subspace_iteration(A,r,q,p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Target rank.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------

    """
    l=r+p
    m, n = A.shape
    omega = np.random.randn(n,l) # Gaussian vectors

    Y = A @ omega
    Q, R = np.linalg.qr(Y,mode='reduced')

    for j in range(1,q+1):
        Y_tilde = A.T @ Q
        Q_tilde, R_tilde = np.linalg.qr(Y_tilde,mode='reduced')

        Y = A @ Q_tilde
        Q, R = np.linalg.qr(Y,mode='reduced')

    return Q

def randomized_subspace_iteration_rSVD(A,r,q,p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Target rank.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------

    """
    start = time.time()

    # Step 1 : Sample column space of A with random P matrix
    Q = randomized_subspace_iteration(A,r,q,p)

    # Step 2: Compute SVD on projected Y = Q.T @ A
    Y = Q.T @ A
    UY, S, VT = np.linalg.svd(Y,full_matrices=0)
    U = Q @ UY

    end = time.time()

    return U, S, VT, (end-start)

def row_extraction_svd(A,r,q,p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Target rank.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------
    Low-rank SVD approximation for A.

    """
    start = time.time()

    # Step 1 : Sample column space of A with random matrix
    m, n = A.shape
    l=r+p
    Omega = np.random.randn(n,l)
    Y = A @ Omega
    for k in range(q):
        Y = A @ (A.T @ Y)

    Q, R = np.linalg.qr(Y,mode='reduced')
    J, proj = interpolative.interp_decomp(Q.T, l, rand=True)

    X = np.zeros((Q.T).shape)
    X[:,J[:l]] = np.eye(l)
    X[:,J[l:]] = proj

    # R, W = rq(A[J[:l],:], mode='economic')
    # Z = X.T @ R
    # U, S, VT_tilde = np.linalg.svd(Z,full_matrices=0)
    # V = W.T @ VT_tilde.T

    # end = time.time()

    # return U, S, V.T, (end-start)
    UY, S, VT = np.linalg.svd(A[J[:l],:],full_matrices=0)
    U = X.T @ UY

    end = time.time()
    return U, S, VT, (end-start)

def reconstruction(A, r, eps, max_iter, target_rank, q, p):
    """

    Parameters
    ----------
    A : numpy array
        Target matrix.
    r : int
        Size of the standard gaussian vector used for error approximation.
    eps : float
        Tolerance for error approximation.
    max_iter : int
        Max number of iterations.
    target_rank : int
        Target rank for classic randomized SVD.
    q : int
        Power iterarions.
    p : int
        Oversampling factor.

    Returns
    -------
    SVD decomposition, error and computation time for classic SVD, randomized SVD, and adaptative randomized SVD

    """
    start = time.time()
    U, S, VT = np.linalg.svd(A) # Full SVD decomposition
    end = time.time()
    ASVD = U[:,:(target_rank+1)] @ np.diag(S[:(target_rank+1)]) @ VT[:(target_rank+1),:] # SVD approximation
    epsSVD = np.linalg.norm(A-ASVD)/np.linalg.norm(A)

    rU, rS, rVT, rt = rSVD(A, target_rank, q, p)
    ArSVD = rU[:,:(target_rank+1)] @ np.diag(rS[:(target_rank+1)]) @ rVT[:(target_rank+1),:] # SVD approximation
    epsrSVD = np.linalg.norm(A-ArSVD)/np.linalg.norm(A)

    # adaptative_rU, adaptative_rS, adaptative_rVT, adaptative_rt = adaptative_range_finder_rSVD(A, r, eps, max_iter, q, p)
    # adaptative_ArSVD = adaptative_rU[:,:(r+1)] @ np.diag(adaptative_rS[:(r+1)]) @ adaptative_rVT[:(r+1),:] # SVD approximation
    # eps_adaptative_rSVD = np.linalg.norm(A-adaptative_ArSVD)/np.linalg.norm(A)

    # iteration_rU, iteration_rS, iteration_rVT, iteration_rt = randomized_subspace_iteration_rSVD(A, target_rank, q, p)
    # iteration_ArSVD = iteration_rU[:,:(r+1)] @ np.diag(iteration_rS[:(r+1)]) @ iteration_rVT[:(r+1),:] # SVD approximation
    # eps_iteration_rSVD = np.linalg.norm(A-iteration_ArSVD)/np.linalg.norm(A)

    row_extraction_rU, row_extraction_rS, row_extraction_rVT, row_extraction_rt = row_extraction_svd(A, target_rank, q, p)
    row_extraction_ArSVD = row_extraction_rU[:,:(target_rank+1)] @ np.diag(row_extraction_rS[:(target_rank+1)]) @ row_extraction_rVT[:(target_rank+1),:] # SVD approximation
    eps_row_extraction_rSVD = np.linalg.norm(A-row_extraction_ArSVD)/np.linalg.norm(A)

    return ASVD, ArSVD, row_extraction_ArSVD, epsSVD, epsrSVD, eps_row_extraction_rSVD, (end-start), rt, row_extraction_rt

#-----------------------------------------------------------------------------#
# MATRIX GENERATORS
#-----------------------------------------------------------------------------#
def fixed_rank_matrix(m, n, rank):
    """

    Parameters
    ----------
    m : int
        Number of rows.
    n : TYPE
        Number of columns.
    rank : int
        Rank of the matrix.

    Returns
    -------
    A : numpy array
        Matrix of size (m,n) & rank=r.
    S : numpy array
        Singular values.

    """
    A = np.random.rand(m,n)
    U, S, VT = np.linalg.svd(A,full_matrices=0)
    S[rank:]=0
    S = (S/S.max())**3  # quickly decaying spectrum
    A = U @ np.diag(S) @ VT
    return A, S

def linear_decaying_spectrum(m, n):
    """

    Parameters
    ----------
    m : int
        Number of rows.
    n : int
        Number of columns

    Returns
    -------
    A : numpy array
        Matrix of size (m,n).
    S : numpy array
        Singular values.

    """
    A = np.random.rand(m,n)
    U, S, VT = np.linalg.svd(A,full_matrices=0)
    S[:m] = np.arange(1,0,-1/n)
    A = U @ np.diag(S) @ VT
    return A, S

#-----------------------------------------------------------------------------#
# POST-PROCESS
#-----------------------------------------------------------------------------#
def time_compare(A, p, q):
    start = time.time()
    U, S, VT = np.linalg.svd(A) # Full SVD decomposition
    end = time.time()
    t_SVD = end-start

    X = []
    T_rSVD = []
    T_extract_rSVD = []
    for i in range(1,501,10):
        X.append(i)

        rU, rS, rVT, rt = rSVD(A, i, q, p)
        T_rSVD.append(rt)

        row_extraction_rU, row_extraction_rS, row_extraction_rVT, row_extraction_rt = row_extraction_svd(A, i, q, p)
        T_extract_rSVD.append(row_extraction_rt)

    T_SVD = [t_SVD for i in range(len(T_rSVD))]

    plt.semilogy(X,T_SVD,'r',label='SVD')
    plt.semilogy(X,T_rSVD,'k',label='rSVD')
    plt.semilogy(X,T_extract_rSVD,'b',label='row_extract_rSVD')

    plt.xlabel("Target rank")
    plt.ylabel("Time (s)")

    plt.legend()
    plt.savefig('time_compare.png', dpi=600)
    plt.show()

def check_adaptative_randomized_range_finder(X, m, n, rank, r, eps, max_iter):
    A, S = fixed_rank_matrix(m, n, rank)
    Q_approx, approx_error_list, true_error_list, j = adaptative_randomized_range_finder(A, r, eps, max_iter)

    print(f"\nReal rank: {rank} | Computed rank: {Q_approx.shape[1]}")

    fig, ax = plt.subplots(1, 1)
    ax.plot(approx_error_list, 'k', label="estimated error")
    ax.plot(true_error_list, 'r', label="true error")
    ax.set(yscale='log', ylabel="error", xlabel='iter', title=f"Real rank: {rank} | Computed rank: {Q_approx.shape[1]}")
    plt.legend()
    plt.savefig('adaptative_randomized_range_finder.png', dpi=600)
    plt.show()

    Q_approx, approx_error_list, true_error_list, j = adaptative_randomized_range_finder(X, r, eps, max_iter)
    fig, ax = plt.subplots(1, 1)
    ax.plot(approx_error_list, 'k', label="estimated error")
    ax.plot(true_error_list, 'r', label="true error")
    ax.set(yscale='log', ylabel="error", xlabel='iter', title=f"Computed rank: {Q_approx.shape[1]}")
    plt.legend()
    plt.savefig('adaptative_randomized_range_finder.png', dpi=600)
    plt.show()

def accuracy_check(A, q, p):
    step = 50
    start = 1

    m, n = A.shape

    U, S, VT = np.linalg.svd(A,full_matrices=0) #SVD decompisition

    error = []
    for r in range(start,start+step):
        rU, rS, rVT, rt = rSVD(A, r, q=0, p=0)
        ArSVD = rU[:,:(r+1)] @ np.diag(rS[:(r+1)]) @ rVT[:(r+1),:] # SVD approximation
        epsrSVD = np.linalg.norm(A-ArSVD,ord=2)
        error.append(epsrSVD)
    plt.plot(np.log10(error), 'b-x', label="$log_{10}(e_{k})$ - $(q,p)=(0,0)$")

    error = []
    for r in range(start,start+step):
        rU, rS, rVT, rt = rSVD(A, r, q=0, p=5)
        ArSVD = rU[:,:(r+1)] @ np.diag(rS[:(r+1)]) @ rVT[:(r+1),:] # SVD approximation
        epsrSVD = np.linalg.norm(A-ArSVD,ord=2)
        error.append(epsrSVD)
    plt.plot(np.log10(error), 'r-x', label="$log_{10}(e_{k})$ - $(q,p)=(0,5)$")

    error = []
    for r in range(start,start+step):
        rU, rS, rVT, rt = rSVD(A, r, q=2, p=5)
        ArSVD = rU[:,:(r+1)] @ np.diag(rS[:(r+1)]) @ rVT[:(r+1),:] # SVD approximation
        epsrSVD = np.linalg.norm(A-ArSVD,ord=2)
        error.append(epsrSVD)
    plt.plot(np.log10(error), 'g-x', label="$log_{10}(e_{k})$ - $(q,p)=(2,5)$")

    plt.plot(np.log10(S[start+1:start+step+1]), 'k-x', label="$log_{10}(\sigma_{k+1})$")

    plt.xlabel("$k$")
    plt.legend()
    plt.savefig('accuracy.png', dpi=600)
    plt.show()

def plot_singular_values(S):
    plt.semilogy(np.diag(S))
    plt.title('Singular values')
    plt.show()

def plot_images(ASVD, ArSVD, row_extraction_ArSVD, target_rank):
    fig, axs = plt.subplots(1,3)

    plt.set_cmap('gray')

    axs[0].imshow(ASVD)
    axs[0].axis('off')
    axs[0].title.set_text('SVD')
    axs[1].imshow(ArSVD)
    axs[1].axis('off')
    axs[1].title.set_text('rSVD')
    axs[2].imshow(row_extraction_ArSVD)
    axs[2].axis('off')
    axs[2].title.set_text('Row extract rSVD')

    fig.tight_layout()
    fig.subplots_adjust(top=0.88)

    fig.savefig(f"jupiter_svd_compare_r_{target_rank}.png", dpi=600)
    plt.show()

def plot_compare(A, r, eps, max_iter, target_rank, q, p):
    ASVD, ArSVD, row_extraction_ArSVD, epsSVD, epsrSVD, eps_row_extraction_rSVD, t, rt, row_extraction_rt = reconstruction(A, r=10, eps=1e-5, max_iter=A.shape[1], target_rank=target_rank, q=q, p=p)
    plot_images(ASVD, ArSVD, row_extraction_ArSVD, target_rank)

def illustrate_power_iterations():
    A, S = linear_decaying_spectrum(m=1000, n=100)

    color_list = np.array([[0,0,2/3],
                           [0,0,1],
                           [0,1/3,1],
                           [0,2/3,1],
                           [0,1,1],
                           [1/3,1,2/3],
                           [2/3,1,1/3],
                           [1,1,0],
                           [1,2/3,0],
                           [1,1/3,0],
                           [1,0,0],
                           [2/3,0,0]])

    plt.plot(S,color='k',label='SVD')

    Y = A
    for q in range(1,6):
        Y = A.T @ Y
        Y = A @ Y
        Uq, Sq, VTq = np.linalg.svd(Y,full_matrices=0)
        plt.plot(Sq, color=tuple(color_list[2*q+1]),label='rSVD, q='+str(q))

    plt.xlabel("$k$")
    plt.legend()
    plt.savefig('power_iteration.png', dpi=600)
    plt.show()

def printInfo(epsSVD, epsrSVD, eps_row_extraction_rSVD, t, rt, row_extraction_rt):
    data = [["Error",epsSVD, epsrSVD, eps_row_extraction_rSVD],
            ["Time (s)",t, rt, row_extraction_rt]]
    print ("\n",tabulate(data, headers=["","SVD","rSVD", "row_extract_rSVD"]))

A = imread('data/jupiter.png')
A = np.mean(A,axis=2) # Convert RGB to grayscale

# img = plt.imshow(A)
# img.set_cmap('gray')
# plt.axis('off')
# plt.show()

target_rank = 1000
q = 2
p = 5

ASVD, ArSVD, row_extraction_ArSVD, epsSVD, epsrSVD, eps_row_extraction_rSVD, t, rt, row_extraction_rt = reconstruction(A, r=10, eps=1e-5, max_iter=A.shape[1], target_rank=target_rank, q=q, p=p)

# printInfo(epsSVD, epsrSVD, eps_row_extraction_rSVD, t, rt, row_extraction_rt)

# check_adaptative_randomized_range_finder(m=1000, n=1000, rank=50, r=20, eps=1e-5, max_iter=200)

# illustrate_power_iterations()

# accuracy_check(A, q, p)

# check_adaptative_randomized_range_finder(A, m=1000, n=500, rank=50, r=10, eps=1e-5, max_iter=1600)

# time_compare(A, p, q)

plot_compare(A, r=10, eps=1e-5, max_iter=A.shape[1], target_rank=target_rank, q=q, p=p)


import tensorflow as tf
import numpy as np
from ops import dense, variable_summaries
import tensorflow.contrib.slim as slim

class CNNModel:
    def __init__(self, sess, CONFIG, lr_decay, lr_step, seq_maxlen):
        self.n_hidden = CONFIG.n_hidden
        self.n_layers = CONFIG.n_layers
        self.n_classes = CONFIG.n_classes
        self.model_dir = CONFIG.model_dir

        self.lr_init = CONFIG.lr_init
        self.lr_decay = lr_decay
        self.lr_step = lr_step
        self.lr_min = CONFIG.lr_min
        self.n_features = CONFIG.n_features
        self.seq_maxlen = seq_maxlen

        # Input params
        self.input = tf.placeholder('float32', [None, self.seq_maxlen, self.n_features, 1])  # [batch_size, seq_length, n_features]
        self.labels = tf.placeholder('float32', [None, self.n_classes])
        self.keep_prob = tf.placeholder('float32')
        self.global_step = tf.Variable(0, trainable=False)

        with slim.arg_scope([slim.conv2d], weights_initializer=tf.contrib.layers.xavier_initializer_conv2d(),
                            weights_regularizer=slim.l2_regularizer(0.0005)):
            with tf.variable_scope('Convolution'):
                net = slim.conv2d(self.input, 16, [1, 3], padding='VALID', scope='conv1')
                net = slim.conv2d(net, 16, [1, 3], padding='VALID', scope='conv2')
                net = slim.max_pool2d(net, [3, 1], stride=3, scope='pool1')
                net = slim.conv2d(net, 32, [1, 2], padding='VALID', scope='conv3')
                net = slim.conv2d(net, 32, [1, 2], padding='VALID', scope='conv4')
                net = slim.max_pool2d(net, [3, 1], stride=3, scope='pool2')
                net = slim.dropout(net, self.keep_prob, scope='Dropout')

        flat = tf.contrib.layers.flatten(net)
        dense_out, dense_w, dense_b = dense(flat, tf.nn.relu, 50, name='dense')
        output = tf.nn.dropout(dense_out, self.keep_prob)

        self.logits, self.output_W, self.output_b = dense(output, None, self.n_classes, name='softmax')
        self.logits = tf.reshape(self.logits, [tf.shape(self.input)[0], self.n_classes])

        # class weights
        class_weights = tf.constant([CONFIG.class_weighting])
        weight_per_label = tf.matmul(self.labels, tf.transpose(class_weights))

        # compute mean weighted loss
        losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.labels)
        self.mean_loss = tf.reduce_mean(tf.multiply(weight_per_label, tf.expand_dims(losses, axis=-1)))

        # run optimization
        self.learning_rate_op = tf.maximum(self.lr_min,
                                           tf.train.exponential_decay(
                                               self.lr_init,
                                               self.global_step,
                                               self.lr_step,
                                               self.lr_decay,
                                               staircase=True))

        self.optimizer = tf.train.AdamOptimizer(self.learning_rate_op)
        self.gradients_vars = self.optimizer.compute_gradients(self.mean_loss)
        self.clipped_gradients_vars = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in self.gradients_vars]
        self.train_op = self.optimizer.apply_gradients(self.clipped_gradients_vars)

        # Evaluate model
        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.labels, 1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

        self.constructSummary(sess)

    def constructSummary(self, sess):
        variable_summaries(self.mean_loss, 'loss')
        variable_summaries(self.accuracy, 'accuracy')
        variable_summaries(self.learning_rate_op, 'learning_rate')
        variable_summaries(self.output_W, 'output_weights')
        variable_summaries(self.gradients_vars[0], 'gradients')
        self.merged = tf.summary.merge_all()
        self.train_writer = tf.summary.FileWriter(self.model_dir + 'train', sess.graph)
        self.val_writer = tf.summary.FileWriter(self.model_dir + 'validation', sess.graph)

    def train(self, sess, batch_input, batch_labels, keep_prob, global_step):

        _, loss, label_err_rate, statistics = sess.run([self.train_op, self.mean_loss, self.accuracy, self.merged],
                                                       feed_dict={self.input: np.expand_dims(batch_input, axis=-1),
                                                            self.labels: batch_labels,
                                                            self.global_step: global_step,
                                                            self.keep_prob: keep_prob})

        self.train_writer.add_summary(statistics, global_step)
        return loss, label_err_rate

    def validate(self, sess, test_input, test_labels, global_step):
        loss, label_err_rate, statistics = sess.run([self.mean_loss, self.accuracy, self.merged],
                                                    feed_dict={self.input: np.expand_dims(test_input, axis=-1),
                                                         self.labels: test_labels,
                                                         self.keep_prob: 1.0})
        self.val_writer.add_summary(statistics, global_step)
        return loss, label_err_rate

    def test(self, sess, test_input, test_labels):

        loss, accuracy, logits = sess.run([self.mean_loss, self.accuracy, self.logits],
                                                feed_dict={self.input: np.expand_dims(test_input, axis=-1),
                                                          self.labels: test_labels,
                                                          self.keep_prob: 1.0})
        confusion_matrix = tf.confusion_matrix(tf.argmax(logits, axis=1),
                                               tf.argmax(test_labels, axis=1)).eval(session=sess)

        return loss, accuracy, confusion_matrix

    def predict(self, sess, pred_input):
        pred_input = pred_input[0].tolist()
        padding_mask = np.zeros(self.n_features).tolist()
        pred_input += [padding_mask for _ in range(self.seq_maxlen - len(pred_input))]
        pred_input = np.expand_dims(np.asarray([pred_input]), axis=-1)
        logits = sess.run(self.logits, feed_dict={self.input: pred_input,
                                                  self.keep_prob: 1.0})
        return logits


from dataclasses import asdict

from app.database.models import User
from app.database.db import CustomDatabase
from app.database.schemas import UserDataclass

class UserRepository:
    def __init__(self, db: CustomDatabase) -> None:
        self.db = db

    async def get(self, data: UserDataclass):
        query = User.select().where(User.c.user_id == data.user_id)
        user_db = await self.db.fetch_one(query)
        if user_db is not None:
            new_data = UserDataclass(
                user_id=user_db.user_id
            )
            return new_data
        return None

    async def add(self, data: UserDataclass):
        query = User.insert()
        await self.db.execute(query=query, values=asdict(data))

    async def upsert(self, data: UserDataclass):
        User = await self.get(data)
        if User is None:
            await self.add(data)
        else:
            await self.update(data)

    async def update(self, data: UserDataclass):
        query = User.update().where(User.c.user_id == data.user_id)
        await self.db.execute(query=query, values=asdict(data))

    async def delete(self, data: UserDataclass):
        query = User.delete().where(User.c.user_id == data.user_id)
        await self.db.execute(query=query)


import os, sys, stat
import socket
import platform
import requests
import tldextract
import pandas as pd
from datetime import datetime
from pandas import ExcelFile
from pandas import ExcelWriter
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

# Check OS and use relevant driver
if platform.system() == 'Windows':
    chrome_driver = '.\\driver\\chromedriver.exe'
elif platform.system() == 'Darwin':
    chrome_driver = './driver/chromedriver_mac64'
else:
    chrome_driver = './driver/chromedriver_linux64'

# Define time format for unique screenshot file names
screenshot_time = datetime.strftime(datetime.now(), ' %Y-%m-%d %H-%M-%S')

# Chrome Driver configuration for taking screenshots
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--window-size=1366,768")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument('--ignore-certificate-errors')

# Ask for screenshots
take_screens = input("Take website screenshots? (Chrome required) [1] For all websites [2] Only for non HTTP 200 responses (for errors) [3] No screenshots: ")

# Ask for file
input_file = './domains.xlsx'
input_column = 'input_urls'
output_file = './results.xlsx'

# File location
src = pd.read_excel(input_file)
column = src[input_column]

# Count number of rows in excel file
countRow = src.shape[0]

# Create empty list
url_list = []
code_list = []
ip_list = []

# Print a message

print("\n" + "=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=  \n" + "=     webr by Nikoloz Kokhreidze | @nikolozsec    = \n" + "=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=  \n")
print("CHECKING {} ITEMS: \n".format(countRow))

# Go throught the File
for i in src.index:

# Separates the gTLD or ccTLD from the registered domain and subdomains of a URL
    domain_extract = tldextract.extract(column[i])
    domain = domain_extract.domain + '.' + domain_extract.suffix
    httpUrl = 'http://' + domain

# Add URLs to the list
    url_list.append(httpUrl)

# Get IP from domain
    try:
        ip = socket.gethostbyname(domain)
        ip_list.append(ip)

# If socket.error, write error in excel
    except socket.error as error:
        ip_list.append("socket error")
        ip = "socket error"

# Try to send request
    try:
        r = requests.get(httpUrl, stream=True)
        status = r.status_code
        code_list.append(status)

# If ConnectionError, write error in excel
    except requests.ConnectionError as error:
        code_list.append("connection error")
        status = "connection error"

# Take screenshots based on user input
    if take_screens == '1':
        driver = webdriver.Chrome(options=chrome_options, executable_path=chrome_driver)
        driver.get(column[i])
        driver.save_screenshot(os.path.join('screenshots', domain + screenshot_time + '.png'))
        driver.close()

    elif take_screens == '2' and status != 200:
        driver = webdriver.Chrome(options=chrome_options, executable_path=chrome_driver)
        driver.get(column[i])
        driver.save_screenshot(os.path.join('screenshots', domain + screenshot_time + '.png'))
        driver.close()

    print(str(domain) + " -  " + str(status) + " - " + str(ip))

# Put output in data
data = {'URL': url_list, 'STATUS': code_list, 'IP': ip_list}

# Create a dataframe and write to new excel file
df = pd.DataFrame(data=data, index=None)
print("\n" + "=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=  \n" + "=                   RESULTS                       = \n" + "=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=  \n" + str(df))
writer = ExcelWriter(output_file)
df.to_excel(writer,'webr', index=False)
writer.save()


# 14. Diseñar un algoritmo que lea un valor correspondiente a un mes y valide si es correcto,
# de lo contrario se debe escribir que es incorrecto y se debe volver a leer hasta que se entre
# bien. Cuando se haya leído bien el mes, se debe escribir el nombre del mes.

try:
    while True:
        valor=int(input("Ingrese el mes: "))
        if valor<1 or valor>12:
            print(f"{valor} no es un mes válido. Debe ingresar un nuevo valor.\n")
        else:
            if valor==1:
                mes="enero"
            elif valor==2:
                mes="febrero"
            elif valor==3:
                mes="marzo"
            elif valor==4:
                mes="abril"
            elif valor==5:
                mes="mayo"
            elif valor==6:
                mes="junio"
            elif valor==7:
                mes="julio"
            elif valor==8:
                mes="agosto"
            elif valor==9:
                mes="septiembre"
            elif valor==10:
                mes="octubre"
            elif valor==11:
                mes="noviembre"
            else:
                mes="diciembre"
            print(f"El mes es {mes}")

except TypeError:
    print("Ingresó un valor inválido como número")


import cv2
import copy

def img_threshold(img, threshold):
    """
    img = image a traiter
    threshold = valeur de seuillage (sur une echelle de gris allant de 0 a 255)
    group = nombre de pixels voisins d'un groupe limite amenant a sa suppression ou non (nettoyage des parasites)
    output = image opencv seuillee et nettoyee
    """
    img_clean = copy.deepcopy(img)

    #Seuillage : attribution de 0 ou 255 a chaque pixel suivant sa valeur par rapport au seuil
    _,img_clean = cv2.threshold(img_clean, threshold, 255, cv2.THRESH_TOZERO)
    _,img_clean = cv2.threshold(img_clean, 0, 255, cv2.THRESH_BINARY)

    return(img_clean)

def cleaner_threshold(img, group):
    """
    img = image a traiter
    group = nombre de voisin
    output = image nettoyee
    """
    new_img = copy.deepcopy(img)

    def voisin(x,y):
        V = []
        if img[y - 1][x] == 255:
            V += [(x,y-1)]
        if img[y + 1][x] == 255:
            V += [(x,y+1)]
        if img[y][x - 1] == 255:
            V += [(x-1,y)]
        if img[y][x + 1] == 255:
            V += [(x+1,y)]
        return(V)

    #Parcours de l'image a la recherche de groupes de pixels isoles (parasites)
    height, width = len(img), len(img[0])
    for x in range (3, width-3):
        for y in range(3, height-3):
            if img[y][x] == 255:
                V = voisin(x,y)
                v = len(V)
                for i,j in V:
                    v += len(voisin(i,j))
                if v <= group:
                    new_img[y][x] = 0
    return(new_img)

def find_threshold(img):
    """
    img = image a traiter
    output = seuillage moyen et renverse
    """
    nb_px = len(img)*len(img[0])
    sum_tot = sum([sum(l) for l in img])
    return( int(sum_tot/nb_px) )


# coding=utf-8
"""Blog feature tests."""

from lib.MobileDriver import MobileDriver
from pageobjects import CountrySelectionPage
from pageobjects import LanguageSelectionPage
from pageobjects import LoginPage
from pageobjects import ShopSelectionPage , TutorialPage , CatalogPage , CartPage
from selenium.webdriver.support.ui import WebDriverWait

import logging
import datetime
from time import mktime

from pytest_bdd import (
    given,
    scenario,
    then,
    when,
)

@scenario('test_scenario1.feature', 'Facebook Login')
def test_Scenario1_FacebookLogin(mobileDriver: MobileDriver):
    print('step 1')

@given('the Zalora App has completed Country and Language selection')
def completeCountryAndLanguageSelection(mobileDriver: MobileDriver):
    """Sets up """
    countrySelectionPage = CountrySelectionPage.CountrySelectionPage(mobileDriver)
    assert countrySelectionPage.exists() , "Country selection page does not exist"
    countrySelectionPage.selectCountry("MALAYSIA")

    languageSelectionPage = LanguageSelectionPage.LanguageSelectionPage(mobileDriver)
    assert languageSelectionPage.exists(), "Language selection page does not exist"
    languageSelectionPage.selectEnglish()

    loginPage = LoginPage.LoginPage(mobileDriver)
    assert loginPage.exists() , "Login page does not exist"

@when('I login with Facebook')
def loginToFacebook( mobileDriver: MobileDriver):
    """Post to Login handler"""
    loginPage = LoginPage.LoginPage(mobileDriver)
    loginPage.clickLoginToFacebook()
    loginPage.facebookLoginFormExists()

@then('I should see language selection page')
def shouldSeeLanguageSelectionPage( mobileDriver: MobileDriver):
    """I should not see the error message."""

    print("")


# num = input("Enter 0: ")

# if int(num)==0:
# 	print("Zero is Entered")
# else:
# 	print("You enter another number.")

# num = input("Enter any number: ")

# if int(num)<=50:
# 	print("Number is less than or equal to 50.")
# else:
# 	print("Number is greater than 50.")

#if else with OR operator
gen = input("Are you a Male (y/n): ")

if gen == 'y' or gen == 'Y':
	print("Gender:Male")
else:
	print("Gender:Female")

age = int(input("Enter your age: "))

if age>=18 and age<=120:
	print("You are eligible for voting.")
else:
	print("You are not eligible for voting.")


import aiohttp
from datetime import date

from bot_settings import config

async def daily_food(auth_token: str):
    headers = {'Authorization': f'Token {auth_token}'}
    today_date = str(date.today())
    async with aiohttp.ClientSession() as session:
        async with session.get(
            f'{config.user_food_api_url}{today_date}/',
            headers=headers,
        ) as response:
            return await response.json()


#Write a Python program to determine if a Python shell
#is executing in 32bit or 64bit mode on OS.

#02_13_21

import sys

max_value = sys.maxsize
#sys.maxsize shows max value a variable type can take depending on 32 or 64-bit platform

bit_32 = 2**31-1 #32-bit platform
bit_64 = 2**63-1 #64-bit platform

if max_value == bit_32:
    print("Your system is 32-bit.")
else:
    print("Your system is 64-bit.")


import datetime
from random import choice

class MonteCarlo(object):
    def __init__(self, board, **kwargs):
        # Takes an instance of a Board and, optionally, some keyword arguments.
        # Initialises the list of game states, and the statistics tables.
        self.board = board
        self.states = []
        seconds = kwargs.get('time', 30)
        self.calculation_time = datetime.timedelta(seconds=seconds)
        self.max_moves = kwargs.get('max_moves', 100)
        self.wins = {}
        self.plys = {}

    def update(self, state):
        # Takes a game state, and appends it to the history.
        self.states.append(state)

    def get_play(self):
        # Causes the AI to calculate the best move from the current game state
        # and return it.
        begin = datetime.datetime.utcnow()
        while datetime.datetime.utcnow() - begin < self.calculation_time:
            self.run_simulation()

    def run_simulation(self):
        # Plays out a 'random' game from the current position, then updates the
        # statistics tables with the result.
        visited_states = set()
        states_copy = self.states[:]
        state = states_copy[-1]
        player = self.board.current_player(state)

        expand = True
        for __ in range(self.max_moves):
            legal = self.board.legal_plays(states_copy)

            play = choice(legal)
            state = self.board.next_state(state, play)
            states_copy.append(state)

            # `player` here refers to the player who moved into that state.
            if expand and (player, state) not in self.plays:
                expand = False
                self.plays[(player, state)] = 0
                self.wins[(player, state)] = 0

            player = self.board.current_player(state)
            winner = self.board.winner(states_copy)
            if winner:
                break

            for player, state in visited_states:
                if (player, state) not in self.plays:
                    continue
                self.plays[(player, state)] += 1
                if player == winner:
                    self.wins[(player, state)] += 1


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan  1 13:59:12 2021

@author: Debbora Leip
"""
import numpy as np
from scipy.special import comb
import scipy.stats as stats
import pickle
import sys

from ModelCode.Auxiliary import _printing
from ModelCode.Auxiliary import _GetDefaults

# %% ############# FUNCTIONS TO GET INPUT FOR FOOD SECURITY MODEL #############

def DefaultSettingsExcept(PenMet = "default",
                          probF = "default",
                          probS = "default",
                          rhoF = "default",
                          rhoS = "default",
                          solv_const = "default",
                          k = "default",
                          k_using = "default",
                          num_crops = "default",
                          yield_projection = "default",
                          sim_start = "default",
                          pop_scenario = "default",
                          risk = "default",
                          N = "default",
                          validation_size = "default",
                          T = "default",
                          seed = "default",
                          tax = "default",
                          perc_guaranteed = "default",
                          ini_fund = "default",
                          food_import = "default",
                          accuracyF_demandedProb = "default",
                          accuracyS_demandedProb = "default",
                          accuracyF_maxProb = "default",
                          accuracyS_maxProb = "default",
                          accuracyF_rho = "default",
                          accuracyS_rho = "default",
                          accuracy_help = "default"):
    """
    Using the default for all settings not specified, this creates a
    dictionary of all settings.

    Parameters
    ----------
    PenMet : "prob" or "penalties", optional
        "prob" if desired probabilities are given and penalties are to be
        calculated accordingly. "penalties" if input penalties are to be used
        directly. The default is defined in ModelCode/DefaultModelSettings.py.
    probF : float, optional
        demanded probability of keeping the food demand constraint (only
        relevant if PenMet == "prob"). The default is defined in
        ModelCode/DefaultModelSettings.py.
    probS : float, optional
        demanded probability of keeping the solvency constraint (only
        relevant if PenMet == "prob"). The default is defined in
        ModelCode/DefaultModelSettings.py.
    rhoF : float or None, optional
        If PenMet == "penalties", this is the value that will be used for rhoF.
        if PenMet == "prob" and rhoF is None, a initial guess for rhoF will
        be calculated in GetPenalties, else this will be used as initial guess
        for the penalty which will give the correct probability for reaching
        food demand. The default is defined in ModelCode/DefaultModelSettings.py.
    rhoS : float or None, optional
        If PenMet == "penalties", this is the value that will be used for rhoS.
        if PenMet == "prob" and rhoS is None, a initial guess for rhoS will
        be calculated in GetPenalties, else this will be used as initial guess
        for the penalty which will give the correct probability for solvency.
        The default is defined in ModelCode/DefaultModelSettings.py.
    solv_const : "on" or "off", optional
        Specifies whether the solvency constraint should be included in the
        model. If "off", probS and rhoS are ignored, and the penalty for
        insolvency is set to zero instead.
    k : int, optional
        Number of clusters in which the area is to be devided.
        The default is defined in ModelCode/DefaultModelSettings.py.
    k_using : "all" or a list of int i\in{1,...,k}, optional
        Specifies which of the clusters are to be considered in the model.
        The default is defined in ModelCode/DefaultModelSettings.py.
    num_crops : int, optional
        The number of crops that are used. The default is defined in
        ModelCode/DefaultModelSettings.py.
    yield_projection : "fixed" or "trend", optional
        If "fixed", the yield distribtuions of the year prior to the first
        year of simulation are used for all years. If "trend", the mean of
        the yield distributions follows the linear trend.
        The default is defined in ModelCode/DefaultModelSettings.py.
    sim_start : int, optional
        The first year of the simulation. The default is defined in
        ModelCode/DefaultModelSettings.py.
    pop_scenario : str, optional
        Specifies which population scenario should be used. "fixed" uses the
        population of the year prior to the first year of the simulation for
        all years. The other options are 'Medium', 'High', 'Low',
        'ConstantFertility', 'InstantReplacement', 'ZeroMigration',
        'ConstantMortality', 'NoChange' and 'Momentum', referring to different
        UN_WPP population scenarios. All scenarios have the same estimates up
        to (including) 2019, scenariospecific predictions start from 2020
        The default is defined in ModelCode/DefaultModelSettings.py.
    risk : int, optional
        The risk level that is covered by the government. Eg. if risk is 0.05,
        yields in the lower 5% quantile of the yield distributions will be
        considered as catastrophic. The default is defined in
        ModelCode/DefaultModelSettings.py.
    N : int, optional
        Number of yield samples to be used to approximate the expected value
        in the original objective function. The default is defined in
        ModelCode/DefaultModelSettings.py.
    validation_size : None or int, optional
        if not None, the objevtice function will be re-evaluated for
        validation with a higher sample size as given by this parameter.
        The default is defined in ModelCode/DefaultModelSettings.py.
    T : int, optional
        Number of years to cover in the simulation. The default is defined in
        ModelCode/DefaultModelSettings.py.
    seed : int, optional
        Seed to allow for reproduction of the results. The default is defined
        in ModelCode/DefaultModelSettings.py.
    tax : float, optional
        Tax rate to be paied on farmers profits. The default is defined in#
        ModelCode/DefaultModelSettings.py.
    perc_guaranteed : float, optional
        The percentage that determines how high the guaranteed income will be
        depending on the expected income of farmers in a scenario excluding
        the government. The default is defined in ModelCode/DefaultModelSettings.py.
    ini_fund : float, optional
        Initial fund size. The default is defined in ModelCode/DefaultModelSettings.py.
    food_import : float, optional
        Amount of food that is imported (and therefore substracted from the
        food demand). The default is defined in ModelCode/DefaultModelSettings.py.
    accuracyF_demandedProb : float, optional
        Accuracy demanded from the food availability probability as share of
        demanded probability (for target prob method). The default is defined in
        ModelCode/DefaultModelSettings.py.
    accuracyS_demandedProb : float, optional
        Accuracy demanded from the solvency probability as share of demanded
        probability (for target prob method). The default is defined in
        ModelCode/DefaultModelSettings.py.
    accuracyF_maxProb : float, optional
        Accuracy demanded from the food demand probability as share of maximum
        probability (for maxProb method). The default is defined in
        ModelCode/DefaultModelSettings.py.
    accuracyS_maxProb : float, optional
        Accuracy demanded from the solvency probability as share of maximum
        probability (for maxProb method). The default is defined in
        ModelCode/DefaultModelSettings.py.
    accuracyF_rho : float, optional
        Accuracy of the food security penalty given thorugh size of the accuracy
        interval: the size needs to be smaller than final rhoF * accuracyF_rho.
        The default is defined in ModelCode/DefaultModelSettings.py.
    accuracyS_rho : float, optional
        Accuracy of the solvency penalty given thorugh size of the accuracy
        interval: the size needs to be smaller than final rhoS * accuracyS_rho.
        The default is defined in ModelCode/DefaultModelSettings.py.
    accuracy_help : float, optional
        If method "MinHelp" is used to find the correct penalty, this defines the
        accuracy demanded from the resulting necessary help in terms of distance
        to the minimal necessary help (given as share of the minimum nevessary
        help). The default is defined in ModelCode/DefaultModelSettings.py.

    Returns
    -------
    settings : dict
        A dictionary that includes all of the above settings.
    """

    # getting defaults for the not-specified settings
    PenMet, probF, probS, rhoF, rhoS, solv_const, k, k_using, \
    num_crops, yield_projection, sim_start, pop_scenario, \
    risk, N, validation_size, T, seed, tax, perc_guaranteed, \
    ini_fund, food_import, accuracyF_demandedProb, accuracyS_demandedProb, \
    accuracyF_maxProb, accuracyS_maxProb, accuracyF_rho, \
    accuracyS_rho, accuracy_help = _GetDefaults(PenMet, probF, probS, rhoF, rhoS,
                solv_const, k, k_using, num_crops, yield_projection,
                sim_start, pop_scenario, risk, N, validation_size, T,
                seed, tax, perc_guaranteed, ini_fund, food_import,
                accuracyF_demandedProb, accuracyS_demandedProb,
                accuracyF_maxProb, accuracyS_maxProb, accuracyF_rho,
                accuracyS_rho, accuracy_help)

    # making sure the current clusters are given as a list
    if type(k_using) is int:
        k_using = [k_using]

    if type(k_using) is tuple:
        k_using = list(k_using)

    if k_using == "all":
        k_using = list(range(1, k + 1))

    k_using_tmp = k_using.copy()

    # This will always be True except when the function is called from
    # GetResultsToCompare() for multiple subsets of clusters
    # (e.g. k_using = [(1,2),(3,6)])
    if sum([type(i) is int for i in k_using_tmp]) == len(k_using_tmp):
        k_using_tmp.sort()

    # check modus (probabilities or penalties given?)
    if PenMet == "penalties":
        probS = None
        probF = None
    elif PenMet != "prob":
        if PenMet != "NotApplicable":
            sys.exit("A non-valid penalty method was chosen (PenMet must " + \
                     "be either \"prob\" or \"penalties\").")

    # create dictionary of settings
    settings =  {"PenMet": PenMet,
                 "probF": probF,
                 "probS": probS,
                 "rhoF": rhoF,
                 "rhoS": rhoS,
                 "solv_const": solv_const,
                 "k": k,
                 "k_using": k_using_tmp,
                 "num_crops": num_crops,
                 "yield_projection": yield_projection,
                 "sim_start": sim_start,
                 "pop_scenario": pop_scenario,
                 "risk": risk,
                 "N": N,
                 "validation_size": validation_size,
                 "T": T,
                 "seed": seed,
                 "tax": tax,
                 "perc_guaranteed": perc_guaranteed,
                 "ini_fund": ini_fund,
                 "import": food_import,
                 "accuracyF_demandedProb": accuracyF_demandedProb,
                 "accuracyS_demandedProb": accuracyS_demandedProb,
                 "accuracyF_maxProb": accuracyF_maxProb,
                 "accuracyS_maxProb": accuracyS_maxProb,
                 "accuracyF_rho": accuracyF_rho,
                 "accuracyS_rho": accuracyS_rho,
                 "accuracy_help": accuracy_help}

    return(settings)

def SetParameters(settings,
                  expected_incomes = None,
                  wo_yields = False,
                  VSS = False,
                  console_output = None,
                  logs_on = None):
    """

    Based on the settings, this sets most of the parameters that are needed as
    input to the model.

    Parameters
    ----------
    settings : dict
        Dictionary of settings as given by DefaultSettingsExcept().
    expected_incomes : np.array of size (len(k_using),)
        The expected income of farmers in a scenario where the government is
        not involved.
    wo_yields : boolean, optional
        If True, the function will do everything execept generating the yield
        samples (and return an empty list as placeholder for the ylds
        parameter). The default is False.
    VSS : boolean, optional
        If True, instead of yield samples only the average yields will be
        returned and all clusters will be indicated as non-catastrophic by
        cat_clusters, as needed to calculate the deterministic solution on
        which the VSS is based. The default is False.
    console_output : boolean, optional
        Specifying whether the progress should be documented thorugh console
        outputs. The default is defined in ModelCode/GeneralSettings.
    logs_on : boolean, optional
        Specifying whether the progress should be documented in a log document.
        The default is defined in ModelCode/GeneralSettings.

    Returns
    -------
    args : dict
        Dictionary of arguments needed as model input.

        - k: number of clusters in which the area is devided.
        - k_using: specifies which of the clusters are to be considered in
          the model.
        - num_crops: the number of crops that are used
        - N: number of yield samples to be used to approximate the
          expected value in the original objective function
        - cat_cluster: np.array of size (N, T, len(k_using)),
          indicating clusters with yields labeled as catastrophic with 1,
          clusters with "normal" yields with 0
        - terminal years: np.array of size (N,) indicating the year in
          which the simulation is terminated (i.e. the first year with a
          catastrophic cluster) for each sample
        - ylds: np.array of size (N, T, num_crops, len(k_using)) of
          yield samples in 10^6t/10^6ha according to the presence of
          catastrophes
        - costs: np array of size (num_crops,) giving cultivation costs
          for each crop in 10^9$/10^6ha
        - demand: np.array of size (T,) giving the total food demand
          for each year in 10^12kcal
        - ini_fund: initial fund size in 10^9$
        - import: given import that will be subtraced from demand in 10^12kcal
        - tax: tax rate to be paied on farmers profits
        - prices: np.array of size (num_crops,) giving farm gate prices
          farmers earn in 10^9$/10^6t
        - T: number of years covered in model
        - guaranteed_income: np.array of size (T, len(k_using)) giving
          the income guaranteed by the government for each year and cluster
          in case of catastrophe in 10^9$, based on expected income
        - crop_cal : np.array of size (num_crops,) giving the calorie content
          of the crops in 10^12kcal/10^6t
        - max_areas: np.array of size (len(k_using),) giving the upper
          limit of area available for agricultural cultivation in each
          cluster
        - probF : float or None, gigving demanded probability of keeping the
          food demand constraint.
        - probS : float or None, giving demanded probability of keeping the
          solvency constraint.
    yield_information : dict
        Dictionary giving additional information on the yields

        - slopes: slopes of the yield trends
        - constants: constants of the yield trends
        - yld_means: average yields
        - residual_stds: standard deviations of the residuals of the yield
          trends
        - prob_cat_year: probability for a catastrophic year given the
          covered risk level
        - share_no_cat: share of samples that don't have a catastrophe within
          the considered timeframe
        - y_profit: minimal yield per crop and cluster needed to not have
          losses
        - share_rice_np: share of cases where rice yields are too low to
          provide profit
        - share_maize_np: share of cases where maize yields are too low to
          provide profit
        - exp_profit: expected profits in 10^9$/10^6ha per year, crop, cluster
    population_information: dict
        Dictionary giving additional information on the population size in the
        model.

        - population : np.array of size (T,) giving estimates of population
          in considered clusters from simulation start to end for given population
          scenario (from UN PopDiv)
        - total_pop_scen : np.array of size (T,) giving estimates of population
          in West Africa from simulation start to end for given population
          scenario (from UN PopDiv)
        - pop_cluster_ratio2015 : np.array of size (len(k_using),) giving the
          share of population of a cluster to total population of West Africa
          in 2015.
    """

    crops = ["Rice", "Maize"]

    # 0. extract settings from dictionary
    k = settings["k"]
    k_using = settings["k_using"]
    num_crops = settings["num_crops"]
    yield_projection = settings["yield_projection"]
    sim_start = settings["sim_start"]
    pop_scenario = settings["pop_scenario"]
    risk = settings["risk"]
    if VSS:
        N = 1
    else:
        N = settings["N"]
    T = settings["T"]
    seed = settings["seed"]
    tax = settings["tax"]
    perc_guaranteed = settings["perc_guaranteed"]
    if expected_incomes is None:
        expected_incomes = np.zeros(len(k_using))

    # 1. get cluster information (clusters given by k-Medoids on SPEI data)
    with open("InputData/Clusters/Clustering/kMediods" + \
                        str(k) + "_PearsonDistSPEI.txt", "rb") as fp:
        clusters = pickle.load(fp)
        costs = pickle.load(fp)

    # 2. calculate total area and area proportions of the different clusters
    with open("InputData/Population/land_area.txt", "rb") as fp:
        land_area = pickle.load(fp)
    cluster_areas = np.zeros(len(k_using))
    for i, cl in enumerate(k_using):
        cluster_areas[i] = np.nansum(land_area[clusters == cl])
    cluster_areas = cluster_areas * 100 # convert sq km to ha

    # 3. Share of population in the area we use (in 2015):
    with open("InputData/Population/GPW_WA.txt", "rb") as fp:
        gridded_pop = pickle.load(fp)[3,:,:]
    with open("InputData/Population/" + \
                  "UN_PopTotal_Prospects_WesternAfrica.txt", "rb") as fp:
        total_pop = pickle.load(fp)
        scenarios = pickle.load(fp)
    total_pop_est_past = total_pop[np.where(scenarios == "Medium")[0][0],:][0:71]
    if pop_scenario == "fixed":
        total_pop_scen = np.repeat(total_pop[np.where(scenarios == "Medium")[0][0],:][(sim_start-1)-1950], T)
    else:
        total_pop_scen = total_pop[np.where(scenarios == pop_scenario)[0][0],:]\
                                    [(sim_start - 1950):(sim_start + T - 1950)]
        total_pop_year_before = total_pop[np.where(scenarios == pop_scenario)[0][0],:]\
                                    [sim_start - 1 - 1950]
    total_pop_UN_2015 = total_pop_est_past[2015-1950]
    cluster_pop = np.zeros(len(k_using))
    for i, cl in enumerate(k_using):
        cluster_pop[i] = np.nansum(gridded_pop[clusters == cl])
    total_pop_GPW = np.sum(cluster_pop)
    cluster_pop_ratio_2015 = cluster_pop/total_pop_UN_2015
    total_pop_ratio_2015 = total_pop_GPW/total_pop_UN_2015
    considered_pop_scen = total_pop_scen * total_pop_ratio_2015 # use 2015 ratio to scale down population scneario to considered area

    # 5. Per person/day demand
    # based on country specific caloric demand from a paper on food waste,
    # averaged based on area. For more detail see DataPreparation_DoNotRun.py
    with open("InputData/Other/AvgCaloricDemand.txt", "rb") as fp:
        ppdemand = pickle.load(fp)

    # 6. cultivation costs of crops
    # average cultivation costs based on literature data for some West
    # African countries (see DataPreparation_DoNotRun for details and
    # sources)
    with open("InputData/Other/CultivationCosts.txt", "rb") as fp:
        costs = pickle.load(fp)
    costs = np.transpose(np.tile(costs, (len(k_using), 1)))

    # 7. Energy value of crops
    with open("InputData/Other/CalorieContentCrops.txt", "rb") as fp:
        crop_cal = pickle.load(fp)

    # 8. Food demand
    # based on the demand per person and day (ppdemand) and assuming no change
    # of per capita daily consumption we use UN population scenarios for West
    # Africa and scale them down to the area we use, using the ratio from 2015
    # (from gridded GPW data)
    demand = ppdemand * 365 * total_pop_scen
    demand = demand * total_pop_ratio_2015
    # in 10^12 kcal
    demand = 1e-12 * demand

    # 9. guaranteed income as share of expected income w/o government
    # if expected income is not given...
    guaranteed_income = np.repeat(expected_incomes[np.newaxis, :], T, axis=0)
    guaranteed_income = perc_guaranteed * guaranteed_income
    # guaraneteed income per person assumed to be constant over time,
    # therefore scale with population size
    if not pop_scenario == "fixed":
        total_pop_ratios = total_pop_scen / total_pop_year_before
        guaranteed_income = (guaranteed_income.swapaxes(0,1) * total_pop_ratios).swapaxes(0,1)

    # 10. prices for selling crops, per crop and cluster (but same over all clusters)
    with open("InputData//Prices/RegionFarmGatePrices.txt", "rb") as fp:
        prices = pickle.load(fp)
    prices = np.transpose(np.tile(prices, (len(k_using), 1)))

    # 11. thresholds for yields being profitable
    y_profit = costs/prices

    # 12. Agricultural Areas:
    # Landscapes of West Africa - A Window on a Changing World:
    # "Between 1975 and 2013, the area covered by crops doubled in West
    # Africa, reaching a total of 1,100,000 sq km, or 22.4 percent, of the
    # land surface."
    # (https://eros.usgs.gov/westafrica/agriculture-expansion)
    # => Approximate max. agricultural area by 22.4% of total cluster land
    #   area (assuming agricultural area is evenly spread over West Africa)
    # Their area extends a bit more north, where agricultural land is
    # probably reduced due to proximity to desert, so in our region the
    # percentage of agricultural might be a bit higher in reality. But
    # assuming evenly spread agricultural area over the area we use is
    # already a big simplification, hence this will not be that critical.
    max_areas = cluster_areas * 0.224
    # in 10^6ha
    max_areas = 1e-6 * max_areas

    # 13. generating yield samples
    # using historic yield data from GDHY
    with open("InputData/YieldTrends/DetrYieldAvg_k" + str(k) + ".txt", "rb") as fp:
         pickle.load(fp) # yields_avg
         pickle.load(fp) # avg_pred
         pickle.load(fp) # residuals
         pickle.load(fp) # residual_means
         residual_stds = pickle.load(fp)
         pickle.load(fp) # fstat
         constants = pickle.load(fp)
         slopes = pickle.load(fp)
         pickle.load(fp) # crops
         pickle.load(fp) # years
    residual_stds = residual_stds[:, [i - 1 for i in k_using]]
    constants = constants[:, [i - 1 for i in k_using]]
    slopes = slopes[:, [i - 1 for i in k_using]]

    # get yield realizations:
    # what is the probability of a catastrophic year for given settings?
    _printing("\nOverview on yield samples", console_output = console_output, logs_on = logs_on)
    prob_cat_year = RiskForCatastrophe(risk, len(k_using))
    _printing("     Prob for catastrophic year: " + \
             str(np.round(prob_cat_year*100, 2)) + "%", \
             console_output = console_output, logs_on = logs_on)
    # create realizations of presence of catastrophic yields and corresponding
    # yield distributions
    np.random.seed(seed)
    cat_clusters, terminal_years, ylds, yld_means = \
          _YieldRealisations(slopes, constants, residual_stds, sim_start, \
                           N, risk, T, len(k_using), num_crops, \
                           yield_projection, VSS, wo_yields)
    # probability to not have a catastrophe
    no_cat = np.sum(terminal_years == -1) / N
    _printing("     Share of samples without catastrophe: " + str(np.round(no_cat*100, 2)), \
              console_output = console_output, logs_on = logs_on)
    # share of non-profitable crops
    if wo_yields:
        share_rice_np = 0
        share_maize_np = 0
    else:
        share_rice_np = np.sum(ylds[:,:,0,:] < y_profit[0,:])/np.sum(~np.isnan(ylds[:,:,0,:]))
        _printing("     Share of cases with rice yields too low to provide profit: " + \
                 str(np.round(share_rice_np * 100, 2)), console_output = console_output, \
                 logs_on = logs_on)
        share_maize_np = np.sum(ylds[:,:,1,:] < y_profit[1,:])/np.sum(~np.isnan(ylds[:,:,1,:]))
        _printing("     Share of cases with maize yields too low to provide profit: " + \
                 str(np.round(share_maize_np * 100, 2)), console_output = console_output, \
                 logs_on = logs_on)
    # in average more profitable crop
    exp_profit = yld_means * prices - costs
    avg_time_profit = np.nanmean(exp_profit, axis = 0)
    more_profit = np.argmax(avg_time_profit, axis = 0) # per cluster
    _printing("     On average more profit (per cluster): " + \
             str([crops[i] for i in more_profit]), \
             console_output = console_output, logs_on = logs_on)
    # in average more productive crop
    avg_time_production = np.nanmean(yld_means, axis = 0)
    more_food = np.argmax(avg_time_production, axis = 0)
    _printing("     On average higher productivity (per cluster): " + \
             str([crops[i] for i in more_food]) + "\n", \
             console_output = console_output, logs_on = logs_on)

    # 14. group output into different dictionaries
    # arguments that are given to the objective function by the solver
    args = {"k": k,
            "k_using": k_using,
            "num_crops": num_crops,
            "N": N,
            "cat_clusters": cat_clusters,
            "terminal_years": terminal_years,
            "ylds": ylds,
            "costs": costs,
            "demand": demand,
            "ini_fund": settings["ini_fund"],
            "import": settings["import"],
            "tax": tax,
            "prices": prices,
            "T": T,
            "guaranteed_income": guaranteed_income,
            "crop_cal": crop_cal,
            "max_areas": max_areas,
            "probF": settings["probF"],
            "probS": settings["probS"]}

    # information not needed by the solver but potentially interesting
    yield_information = {"slopes": slopes,
             "constants": constants,
             "yld_means": yld_means,
             "residual_stds": residual_stds,
             "prob_cat_year": prob_cat_year,
             "share_no_cat": no_cat,
             "y_profit": y_profit,
             "share_rice_np": share_rice_np,
             "share_maize_np": share_maize_np,
             "exp_profit": exp_profit}

    population_information = {"population": considered_pop_scen,
                              "total_pop_scen": total_pop_scen, # full area WA
                              "pop_cluster_ratio2015": cluster_pop_ratio_2015}

    return(args, yield_information, population_information)

def RiskForCatastrophe(risk, num_clusters):
    """
    The risk level covered by the government is an input parameter to the
    model. The resulting probability for a catastrophic year depends on the
    number of clusters used by the model.

    Parameters
    ----------
    risk : int
        The risk level that is covered by the government. Eg. if risk is 0.05,
        yields in the lower 5% quantile of the yield distributions will be
        considered as catastrophic.
    num_clusters : int
        The number of clusters that are included.

    Returns
    -------
    res : The probability for a year to be catastrophic.

    """
    res = 0
    for i in range(1, num_clusters + 1):
        res += comb(num_clusters, i) * (risk**i) * (1-risk)**(num_clusters-i)

    return(res)

def _YieldRealisations(yld_slopes, yld_constants, resid_std, sim_start, N, \
                      risk, T, num_clusters, num_crops, \
                      yield_projection, VSS = False, wo_yields = False):
    """
    Depending on the risk level the occurence of catastrophes is generated,
    which is then used to generate yield samples from the given yield
    distributions.

    Parameters
    ----------
    yld_slopes : np.array of size (num_crops, len(k_using))
        Slopes of the yield trends.
    yld_constants : np.array of size (num_crops, len(k_using))
        Constants of the yield trends.
    resid_std : np.array of size (num_crops, len(k_using))
        Standard deviations of the residuals of the yield trends.
    sim_start : int
        The first year of the simulation.
    N : int
        Number of yield samples to be used to approximate the expected value
        in the original objective function.
    risk : int
        The risk level that is covered by the government. Eg. if risk is 0.05,
        yields in the lower 5% quantile of the yield distributions will be
        considered as catastrophic.
    T : int
        Number of years to cover in the simulation.
    num_clusters : int
        Number of clusters considered by model (corresponding to len(k_using))
    num_crops : int
        The number of crops that are used.
    yield_projection : "fixed" or "trend"
        If "fixed", the yield distribtuions of the year prior to the first
        year of simulation are used for all years. If "trend", the mean of
        the yield distributions follows the linear trend.
    VSS : boolean, optional
        If True, all clusters will be indicated as non-catastrophic, as needed
        to calculate the deterministic solution on which the VSS is based.
        The default is False.
    wo_yields : boolean, optional
        If True, the function will do everything execept generating the yield
        samples (and return an empty list as placeholder for the ylds
        parameter). The default is False.

    Returns
    -------
    cat_cluters : np.array of size (N, T, len(k_using))
        indicating clusters with yields labeled as catastrophic with 1,
        clusters with "normal" yields with 0
    terminal_years : np.array of size (N,)
        indicating the year in which the simulation is terminated (i.e. the
        first year with a catastrophic cluster) for each sample, with 0 for
        catastrophe in first year, 1 for second year, etc. If no catastrophe
        happens before T, this is indicated by -1
    ylds : np.array of size (N, T, num_crops, len(k_using))
        yield samples in 10^6t/10^6ha according to the presence of
        catastrophes
    yld_means : np.array of size (T, num_crops, len(k_using))
        Average yields in 10^6t/10^6ha.

    """

    # generating catastrophes
    cat_clusters, terminal_years = _CatastrophicYears(risk, \
                                    N, T, num_clusters, VSS)

    # generating yields according to catastrophes
    ylds, yld_means = _ProjectYields(yld_slopes, yld_constants, resid_std, sim_start, \
                             N, cat_clusters, terminal_years, T, risk, \
                             num_clusters, num_crops, yield_projection, \
                             VSS, wo_yields)

    # set negative yields to zero
    if not wo_yields:
        np.seterr(invalid='ignore') # comparing with np.nans leads to annoying warnings, so we turn them off
        ylds[ylds<0] = 0
        np.seterr(invalid='warn')

    return(cat_clusters, terminal_years, ylds, yld_means)

def _CatastrophicYears(risk, N, T, num_clusters, VSS):
    """
    Given the risk level that is to be covered, this creates a np.array
    indicating which clusters should be catastrophic for each year.

    Parameters
    ----------
    risk : int
        The risk level that is covered by the government. Eg. if risk is 0.05,
        yields in the lower 5% quantile of the yield distributions will be
        considered as catastrophic.
    N : int
        Number of yield samples to be used to approximate the expected value
        in the original objective function.
    T : int
        Number of years to cover in the simulation.
    num_clusters : int
        Number of clusters clusters that are considered
    VSS : boolean
        If True, all clusters will be indicated as non-catastrophic, and
        average yields will be returned instead of yield samples, as needed
        to calculate the deterministic solution on which the VSS is based.

    Returns
    -------
    cat_cluters : np.array of size (N, T, len(k_using))
        indicating clusters with yields labeled as catastrophic with 1,
        clusters with "normal" yields with 0
    terminal_years : np.array of size (N,)
        indicating the year in which the simulation is terminated (i.e. the
        first year with a catastrophic cluster) for each sample

    """

    if VSS:
        cat_clusters = np.zeros((N, T, num_clusters))
        terminal_years = np.repeat(-1, N)
        return(cat_clusters, terminal_years)

    # generating uniform random variables between 0 and 1 to define years
    # with catastrophic cluster (catastrohic if over threshold)
    threshold = 1 - risk
    cat_clusters = np.random.uniform(0, 1, [N, T, num_clusters])
    cat_clusters[cat_clusters > threshold] = 1
    cat_clusters[cat_clusters <= threshold] = 0

    # year is catastrophic if min. one cluster is catastrophic
    cat_years = np.sum(cat_clusters, axis = 2)
    cat_years[cat_years < 1] = 0
    cat_years[cat_years >= 1] = 1

    # terminal year is first catastrophic year. If no catastrophic year before
    # T, we set terminal year to -1
    terminal_years = np.sum(cat_years, axis = 1)
    for i in range(0, N):
        if terminal_years[i] == 0:
            terminal_years[i] = -1
        else:
            terminal_years[i] = np.min(np.where(cat_years[i, :] == 1))
    return(cat_clusters, terminal_years)

def _ProjectYields(yld_slopes, yld_constants, resid_std, sim_start, \
                  N, cat_clusters, terminal_years, T, risk, \
                  num_clusters, num_crops, yield_projection, \
                  VSS = False, wo_yields = False):
    """
    Depending on the occurence of catastrophes yield samples are generated
    from the given yield distributions.

    Parameters
    ----------
    yld_slopes : np.array of size (num_crops, len(k_using))
        Slopes of the yield trends.
    yld_constants : np.array of size (num_crops, len(k_using))
        Constants of the yield trends.
    resid_std : np.array of size (num_crops, len(k_using))
        Standard deviations of the residuals of the yield trends.
    sim_start : int
        The first year of the simulation.
    N : int
        Number of yield samples to be used to approximate the expected value
        in the original objective function.
    cat_cluters : np.array of size (N, T, len(k_using))
        indicating clusters with yields labeled as catastrophic with 1,
        clusters with "normal" yields with 0
    terminal_years : np.array of size (N,)
        indicating the year in which the simulation is terminated (i.e. the
        first year with a catastrophic cluster) for each sample, with 0 for
        catastrophe in first year, 1 for second year, etc. If no catastrophe
        happens before T, this is indicated by -1
    T : int
        Number of years to cover in the simulation.
    risk : int
        The risk level that is covered by the government. Eg. if risk is 0.05,
        yields in the lower 5% quantile of the yield distributions will be
        considered as catastrophic.
    num_clusters : int
        Number of clusters considered by model (corresponding to len(k_using))
    num_crops : int
        The number of crops that are used.
    yield_projection : "fixed" or "trend"
        If "fixed", the yield distribtuions of the year prior to the first
        year of simulation are used for all years. If "trend", the mean of
        the yield distributions follows the linear trend.
    VSS : boolean, optional
        If True, average yields will be returned instead of yield samples, as
        needed to calculate the deterministic solution on which the VSS is
        based. The default is False.
    wo_yields : boolean, optional
        If True, the function will do everything execept generating the yield
        samples (and return an empty list as placeholder for the ylds
        parameter). The default is False.

    Returns
    -------
    ylds : np.array of size (N, T, num_crops, len(k_using))
        yield samples in 10^6t/10^6ha according to the presence of
        catastrophes. If wo_yields = True, this is an empty list.
    yld_means : np.array of size (T, num_crops, len(k_using))
        Average yields in 10^6t/10^6ha.

    """
    # project means of yield distributions (either repeating fixed year or by
    # using trend)
    if yield_projection == "fixed": # for fixed we use the year before start
        year_rel = (sim_start - 1) - 1981
        yld_means = yld_constants + year_rel * yld_slopes
        yld_means = np.repeat(yld_means[np.newaxis, :, :], T, axis = 0)

    elif yield_projection == "trend":
        year_rel = sim_start - 1981
        years = np.transpose(np.tile(np.array(range(year_rel, year_rel + \
                                        T)), (num_clusters, num_crops, 1)))
        yld_means = yld_constants + years * yld_slopes
    resid_std = np.repeat(resid_std[np.newaxis,:, :], T, axis=0)

    # needed for VSS
    if VSS == True:
        return(np.expand_dims(yld_means, axis = 0), yld_means)

    if wo_yields:
        return([], yld_means)

    # initializing yield array
    ylds = np.empty([N, T, num_crops, num_clusters])
    ylds.fill(np.nan)

    # calculating quantile of standard normal distribution corresponding to
    # the catastrophic yield quantile for truncnorm fct
    quantile_low = stats.norm.ppf(risk, 0, 1)

    # generating yields: for catastrophic clusters from lower quantile, for
    # normal years from upper quantile. Dependence between crops, i.e. if
    # cluster is catastrophic, both crops get yields from lower quantile of
    # their distributions
    for run in range(0, N):
        if int(terminal_years[run]) == -1:
            ylds[run, :, :, :] = stats.truncnorm.rvs(quantile_low, np.inf, \
                                                     yld_means, resid_std)
        else:
            term_year = int(terminal_years[run])
            ylds[run, :(term_year+1), :, :] = \
                stats.truncnorm.rvs(quantile_low, np.inf, \
                                    yld_means[:(term_year+1), :, :], \
                                    resid_std[:(term_year+1), :, :])
            for cl in range(0, num_clusters):
                if cat_clusters[run, term_year, cl] == 1:
                    ylds[run, term_year, :, cl] = \
                        stats.truncnorm.rvs(- np.inf, quantile_low, \
                                            yld_means[term_year, :, cl], \
                                            resid_std[term_year, :, cl])

    return(ylds, yld_means)


class Node:

    def __init__(self, val, next = None):
        self.val = val
        self.next = next

def kthLast(node, k):

    count = 0
    var = node
    current = node

    while current:

        count += 1
        if count > k:
            var = var.next
        current = current.next

    return var.val

GitHubProjects/Main Folder/Coding Projects/DailyCodeWeeks/Week 4/task26.py


from typing import Optional
import re
import sys

import tuwien.tiss
import tuwien.colab
import tuwien.rdb

NAME_REPLACE = re.compile(r'[^A-Za-z0-9äüößÄÜÖẞ]')

class Room:
    name: str
    suffix: Optional[str]
    name_short: Optional[str]
    alt_name: Optional[str]
    room_codes: [str]
    tiss_code: Optional[str]
    type: Optional[str]
    parent_name: Optional[str]
    area: Optional[int]
    capacity: Optional[int]
    comment: Optional[str]

    def __init__(self, name: str, room_code: str, tiss_code: str = None, room_type: str = None,
                 area: int = None, capacity: int = None, comment: str = None):
        self.name = re.sub(r' *- +[A-Z-]*$', '', name.split(' - Achtung!')[0]).split(' - mündl.')[0].replace(',', '')
        self.name = self.name.replace('HS', 'Hörsaal')
        self.name = self.name\
            .replace('Sem.R.', 'Seminarraum')\
            .replace('Sem. R.', 'Seminarraum')\
            .replace('Sem.', 'Seminarraum')\
            .replace('Sem ', 'Seminarraum ')
        self.name = self.name.replace('AI', 'Atominstitut')
        self.suffix = None
        self.name_short = None
        self.alt_name = None
        self.room_codes = [room_code.replace(' ', '')]
        self.tiss_code = tiss_code
        self.type = room_type
        if self.type is None:
            if 'hörsaal' in self.name.lower() or 'Audi. Max.' in self.name or 'Atrium' in self.name:
                self.type = 'lecture_hall'
            elif 'seminarraum' in self.name.lower():
                self.type = 'seminar_room'
            elif 'projektraum' in self.name.lower():
                self.type = 'project_room'
            elif 'lab' in self.name.lower():
                self.type = 'lab'
            elif 'zeichensaal' in self.name.lower():
                self.type = 'drawing_room'
            elif 'büro' in self.name.lower():
                self.type = 'office'
            elif self.name.startswith('PC') or self.name.startswith('EDV') or self.name.startswith('CAD'):
                self.type = 'lab'
        self.parent_name = None
        self.area = area
        self.capacity = capacity
        self.comment = comment

if __name__ == '__main__':
    print('Fetching coLAB rooms...', file=sys.stderr)
    colab_rooms = [
        Room(room.name, room.id, comment='colab')
        for room in tuwien.colab.get_rooms()
    ]
    print('Successfully fetched coLAB rooms', file=sys.stderr)

    print('Fetching TISS rooms...', file=sys.stderr)
    s = tuwien.tiss.Session()
    tiss_rooms = [
        Room(room.name, room.global_id, tiss_code=room.id, capacity=room.capacity, comment='tiss')
        for room in s.rooms.values()
    ]
    print('Successfully fetched TISS rooms', file=sys.stderr)

    print('Fetching RDB rooms...', file=sys.stderr)
    rdb_rooms = [
        Room(room.name, room.id, room_type=room.type, area=room.area, comment='rdb')
        for room in tuwien.rdb.get_rooms()
    ]
    print('Successfully fetched RDB rooms', file=sys.stderr)

    print('Processing rooms...', file=sys.stderr)
    rooms: [Room] = colab_rooms + tiss_rooms + rdb_rooms
    rooms.sort(key=lambda room: (room.room_codes[0][0] + NAME_REPLACE.sub('', room.name)).lower())

    proc = [rooms[0]]
    for room in rooms[1:]:
        last = proc[-1]
        ln = NAME_REPLACE.sub('', last.name.split('|')[0].lower())
        rn = NAME_REPLACE.sub('', room.name.lower())
        if last.room_codes[0] == room.room_codes[0] and (room.tiss_code is None or last.tiss_code is None) and \
                (ln.startswith(rn) or rn.startswith(ln)):
            last.area = last.area or room.area
            last.capacity = last.capacity or room.capacity
            last.tiss_code = last.tiss_code or room.tiss_code
            last.comment = last.comment + ' ' + room.comment
            if last.name != room.name:
                last.name = last.name + ' | ' + room.name
            last.type = last.type or room.type
            if last.type != room.type and (last.type is not None and room.type is not None):
                if room.name == 'Aufbaulabor':
                    last.type = 'drawing_room'
        else:
            proc.append(room)
    proc.sort(
        key=lambda room:
        ((room.type or "other")[:3] +
         room.room_codes[0][0] +
         re.sub(r'[0-9]+', lambda m: ('0000' + m.group(0))[-4:], NAME_REPLACE.sub('', room.name))
         ).lower()
    )
    print('Finished processing rooms', file=sys.stderr)

    print('name,suffix,name_short,alt_name,room_codes,tiss_code,type,parent_name,area,capacity,comment')
    for room in proc:
        print(f'{room.name},{room.suffix or ""},{room.name_short or ""},{room.alt_name or ""},'
              f'{" ".join(room.room_codes)},{room.tiss_code or ""},{room.type or "other"},{room.parent_name or ""},'
              f'{room.area or ""},{room.capacity or ""},{room.comment or ""}')


from __future__ import print_function

import sys

import numpy as np
import tensorflow as tf

KERNEL_SIZE=3

class Vgg19Model:
    def __init__(self, parameter_dict_path):
        self.m_parameter_dict=np.load(parameter_dict_path, encoding='latin1').item()
        print('load pre-trained parameter from %s' % parameter_dict_path)

    def _load_layer_parameter(self, name):
        if name in self.m_parameter_dict:
            parameter = self.m_parameter_dict[name]
            weight =  tf.constant(parameter[0], name='weight')
            bias =  tf.constant(parameter[1], name='bias')
            return weight, bias
        else:
            print('Cannot find pre-trained parameter "%s"' % name, file=sys.stderr)
            sys.exit()

    def _build_convolution_layer(self, layer_name, input_tensor):
        with tf.variable_scope(layer_name):
            weight, bias = self._load_layer_parameter(layer_name)
            convolution = tf.nn.conv2d(input_tensor, weight, strides=[1, 1, 1, 1], padding='SAME')
            pre_activation = tf.nn.bias_add(convolution, bias)
            relu = tf.nn.relu(pre_activation)
            return relu

    def _build_fully_connect_layer(self, layer_name, input_tensor):
        if tf.rank(input_tensor) != 2:
            dimension_product= 1
            for dim in input_tensor.get_shape().as_list()[1:]:
                dimension_product *= dim
            input_tensor = tf.reshape(input_tensor, [-1, dimension_product])
        with tf.variable_scope(layer_name):
           weight, bias = self._load_layer_parameter(layer_name)
           return tf.nn.bias_add(tf.matmul(input_tensor, weight), bias)

    def _build_convolution_block(self, block_prefix, input_tensor, stack):
        assert stack > 0
        tensor = input_tensor
        for i in xrange(1, stack+1):
            tensor = self._build_convolution_layer('%s_%d' % (block_prefix, i), tensor)
        return tensor

    def _build_max_pool_layer(self, input_tensor):
        output = tf.nn.max_pool(input_tensor, [1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
        print(output.get_shape().as_list())
        return output

    def build_model(self, input_tensor):
        conv_1 = self._build_convolution_block('conv1', input_tensor, 2)
        pool_1 = self._build_max_pool_layer(conv_1)
        conv_2 = self._build_convolution_block('conv2', pool_1, 2)
        pool_2 = self._build_max_pool_layer(conv_2)
        conv_3 = self._build_convolution_block('conv3', pool_2, 4)
        pool_3 = self._build_max_pool_layer(conv_3)
        conv_4 = self._build_convolution_block('conv4', pool_3, 4)
        pool_4 = self._build_max_pool_layer(conv_4)
        conv_5 = self._build_convolution_block('conv5', pool_4, 4)
        pool_5 = self._build_max_pool_layer(conv_5)

        fc_6 = self._build_fully_connect_layer('fc6', pool_5)
        relu_6 = tf.nn.relu(fc_6)
        fc_7 = self._build_fully_connect_layer('fc7', relu_6)
        relu_7 = tf.nn.relu(fc_7)
        fc_8 = self._build_fully_connect_layer('fc8', relu_7)
        softmax = tf.nn.softmax(fc_8)
        return softmax


'''
File name: ctc_chroma/features.py
Author: Frank Zalkow
Date: 2020
License: MIT

This file is part of the following repository:
  https://github.com/fzalkow/ctc_chroma

If you use code of this reposiory, please cite:
  Frank Zalkow and Meinard Müller:
  Using Weakly Aligned Score–Audio Pairs to Train Deep Chroma Models for Cross-Modal Music Retrieval.
  In Proceedings of the International Society for Music Information Retrieval Conference, Montréal, Canada, 2020.
'''

import os
import argparse
import glob
import multiprocessing as mp

import librosa
from tqdm import tqdm
import numpy as np

def get_hcqt_params():
    """Hack to always use the same parameters :)
    Function from https://github.com/rabitt/ismir2017-deepsalience"""
    bins_per_octave = 36
    n_octaves = 6
    harmonics = [0.5, 1, 2, 3, 4, 5]
    sr = 22050
    fmin = 32.7
    hop_length = 256
    return bins_per_octave, n_octaves, harmonics, sr, fmin, hop_length

def compute_hcqt(audio_fpath):
    """Compute the harmonic CQT from a given audio file
    Function from https://github.com/rabitt/ismir2017-deepsalience"""
    (bins_per_octave, n_octaves, harmonics,
     sr, f_min, hop_length) = get_hcqt_params()
    y, fs = librosa.load(audio_fpath, sr=sr)

    cqt_list = []
    shapes = []
    for h in harmonics:
        cqt = librosa.cqt(y, sr=fs, hop_length=hop_length, fmin=f_min*float(h), n_bins=bins_per_octave*n_octaves,
                          bins_per_octave=bins_per_octave)
        cqt_list.append(cqt)
        shapes.append(cqt.shape)

    shapes_equal = [s == shapes[0] for s in shapes]
    if not all(shapes_equal):
        min_time = np.min([s[1] for s in shapes])
        new_cqt_list = []
        for i in range(len(cqt_list)):
            new_cqt_list.append(cqt_list[i][:, :min_time])
        cqt_list = new_cqt_list

    log_hcqt = ((1.0 / 80.0) * librosa.core.amplitude_to_db(np.abs(np.array(cqt_list)), ref=np.max)) + 1.0

    return log_hcqt

def get_time_grid(n_time_frames):
    """Get the hcqt time grid
    Function from https://github.com/rabitt/ismir2017-deepsalience"""
    (_, _, _, sr, _, hop_length) = get_hcqt_params()
    time_grid = librosa.core.frames_to_time(range(n_time_frames), sr=sr, hop_length=hop_length)
    return time_grid

def get_freq_grid():
    """Get the hcqt frequency grid
    Function from https://github.com/rabitt/ismir2017-deepsalience"""
    (bins_per_octave, n_octaves, _, _, f_min, _) = get_hcqt_params()
    freq_grid = librosa.cqt_frequencies(bins_per_octave*n_octaves, f_min, bins_per_octave=bins_per_octave)
    return freq_grid

def compute_and_save_hcqt(fn_wav, fn_out, feature_rate):
    hcqt, times, freqs = compute_hcqt_median(fn_wav, feature_rate)

    # interpolate times
    eps = np.finfo(float).eps
    new_times = np.arange(times[0], times[-1] + eps, 1.0/feature_rate)  # add eps to include last time point

    hcqt_interpolated = median_bin_salience(hcqt, times, new_times, axis=-1)

    np.savez_compressed(fn_out, f_pitch=hcqt_interpolated, f_pitch_ax_time=new_times, f_pitch_ax_freq=freqs)

def compute_hcqt_median(fn_wav, feature_rate):
    hcqt = compute_hcqt(fn_wav)
    times = get_time_grid(len(hcqt[0][0]))
    freqs = get_freq_grid()

    # interpolate times
    eps = np.finfo(float).eps
    new_times = np.arange(times[0], times[-1] + eps, 1.0/feature_rate)  # add eps to include last time point

    return median_bin_salience(hcqt, times, new_times, axis=-1), new_times, freqs

def median_bin_salience(salience, old_times, new_times, axis=1):

    idx = []
    for i in range(len(new_times)):
        start_time = new_times[i]
        idx.append(np.searchsorted(old_times, start_time))
    idx.append(len(old_times))

    new_X = librosa.util.sync(salience, idx, aggregate=np.median, pad=False, axis=axis)

    return new_X

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--indir')
    parser.add_argument('-o', '--outdir')
    parser.add_argument('--rate', help='Frame Rate', type=int, default=25)
    parser.add_argument('--mode', help='DEVELOP or PRODUCTION', default='PRODUCTION')
    args = parser.parse_args()

    # python 00_extract_features_bittner/extract_features_input.py -i
    # /home/fzalkow/mountpoints/groupmm/Projects/BM/Subsets/barlow-morgenstern/03_BM-medium/data_WAV_WCM
    # -o data/BM_hcqt25 --time-binning median --rate 25

    indir = args.indir
    feature_rate = args.rate
    outdir = args.outdir
    assert os.path.exists(indir)
    assert args.mode in ['DEVELOP', 'PRODUCTION']
    os.makedirs(outdir, exist_ok=True)

    fn_audios = sorted(glob.glob(os.path.join(indir, '*.wav')))

    if args.mode == 'DEVELOP':
        base = os.path.splitext(os.path.basename(fn_audios[0]))[0]
        fn_new = os.path.join(outdir, base + '.npz')
        compute_and_save_hcqt(fn_audios[0], fn_new, feature_rate)

    if args.mode == 'PRODUCTION':
        n_workers = 16  # mp.cpu_count()
        pool = mp.Pool(n_workers)
        amount_of_jobs = len(fn_audios)
        i = 0

        with tqdm(total=amount_of_jobs, desc='Execute jobs', dynamic_ncols=True) as pbar:

            for fn_audio in fn_audios:

                base = os.path.splitext(os.path.basename(fn_audio))[0]
                fn_new = os.path.join(outdir, base + '.npz')
                if not os.path.exists(fn_new):
                    pool.apply_async(compute_and_save_hcqt, args=(fn_audio, fn_new, feature_rate),
                                     callback=lambda _: pbar.update(),
                                     error_callback=lambda x: tqdm.write(str(x)))
                else:
                    pbar.update()
            pool.close()  # no more jobs will be added
            pool.join()


#!/opt/local/bin/python3.5 -tt

# Disregarding carons (ǧ, ȟ and ǩ), these are all of the Nishnaabe
# graphemes:
# - ' a aa b ch d e g h i ii j k m mb
# n nd ng nh nj ns ny nz nzh o oo p
# s sh shk shp sht sk sp t w y z zh
#
#
# Including carons (ǧ, ȟ and ǩ), these are all of the Nishnaabe
# graphemes:
# - ' a aa b ch d e g ǧ h ȟ i ii j k ǩ m mb
# n nd ng nǧ nh nj ns ny nz nzh o oo p
# s sh shk shǩ shp sht sk sp sǩ t w y z zh

def split_into_graphemes(nishchars):
    '''This function splits a Nishnaabe word into its
    respective constituent graphemes. It accepts a string of
    Nishnaabe characters (nishchars) and returns a list of
    graphemes (collectedgraphemes).'''

    # List of possible single-letter-graphemes in Biigtigong's dialect
    # of the Nishnaabe language.
    singlechargraphemes = [
        "-", "'", "a", "b", "d", "e", "g", "ǧ", "h", "ȟ", "i", "j",
        "k", "ǩ",  "l", "m", "n", "o", "p", "s", "t", "w", "y", "z"]

    # List of possible two-letter-graphemes in Biigtigong's dialect
    # of the Nishnaabe language.
    digraphs = [
        "aa", "ch", "ii", "mb", "nd", "ng", "nǧ", "nh", "nj",
        "ns", "ny", "nz", "oo", "sh", "sk", "sǩ", "sp", "zh"]

    # List of possible three-letter-graphemes in Biigtigong's dialect
    # of the Nishnaabe language.
    trigraphs = [
        "nzh", "shk", "shǩ", "shp", "sht"]

    # Convert the Nishnaabe word, nishchars, to all-lowercase
    nishchars = nishchars.lower()

    # Prepare an empty list to store all of the graphemes of the
    # Nishnaabe word nishchars.
    collectedgraphemes = []

    # Calculate the length (in characters) of the Nishnaabe word
    # which will be iterated through.
    numofnishchars = len(nishchars)

    # Extend the length of Nishnaabe word nishchars to prevent an
    # "IndexError: string index out of range"
    # when we iterate past the last character of the Nishnaabe word in
    # our while-loop. This nonsense string needs to have a length at
    # least as large as one character shorter than the length of the
    # longest possible grapheme. (Since the longest possible grapheme is
    # currently three characters, we add one less than three characters
    # to the end of nishchars.) The characters which make up this
    # nonsense string need to be characters which are not possible
    # occurrences within the Nishnaabe orthography of Biigtigong's dialect.
    nishchars += "qq"

    # Initialize our counter with which we'll iterate through the
    # characters of the Nishnaabe word nishchars.
    i = 0

    # Iterate through the Nishnaabe word, nishchars, in order to split
    # it into the graphemes of which nishchars consists.
    while (i < numofnishchars):

        # From the current counter, i, with which we're iterating
        # through nishchars, we create three quick and convenient strings
        # with which to do some querying in the upcoming lines of code.
        queried1char = nishchars[i]
        queried2chars = nishchars[i] + nishchars[i + 1]
        queried3chars = nishchars[i] + nishchars[i + 1] + nishchars[i + 2]

        # Query the list of possible Nishnaabe three-letter-graphemes for
        # the existence of the currently-pointed-to three-character-sequence
        # within the Nishnaabe word nishchars. If the three-character-
        # sequence is found in the trigraphs list, then we add that
        # three-character-sequence to our collectedgraphemes list.
        if queried3chars in trigraphs:
            collectedgraphemes.append(queried3chars)
            i += 3

        # Query the list of possible Nishnaabe two-letter-graphemes for
        # the existence of the currently-pointed-to two-character-sequence
        # within the Nishnaabe word nishchars. If the two-character-
        # sequence is found in the digraphs list, then we add that
        # two-character-sequence to our collectedgraphemes list.
        elif queried2chars in digraphs:
            collectedgraphemes.append(queried2chars)
            i += 2

        # Query the list of possible Nishnaabe one-letter-graphemes for
        # the existence of the currently-pointed-to one-character-sequence
        # within the Nishnaabe word nishchars. If the one-character-
        # sequence is found in the singlechargraphemes list, then we add
        # that one-character-sequence to our collectedgraphemes list.
        elif queried1char in singlechargraphemes:
            collectedgraphemes.append(queried1char)
            i += 1

        # Alert user of an unrecognizable character within the Nishnaabe
        # word currently being worked on.
        else:
            print('I am unable to recognize the following character:')
            print('"' + str(nishchars[i]) + '"')
            print('at position #' + str(i + 1))
            print('in the word: "' + nishchars)
            break

    # Return, as list collectedgraphemes, the constituent split-up
    # graphemes of the Nishnaabe word nishchars.
    return collectedgraphemes

##############################
# Begin main processing loop.
##############################

# A sample list of Nishnaabe words to split into graphemes
nishsamples = [
    "nishnaabe",
    "boozhoo",
    "aaniin",
    "waabndaan",
    "en'goons",
    "biiwaan'goonh",
    "gnoon'diwag",
    "gaa-gii-ni-zhaang",
    "endgwen",
    "gzaaghin",
    "wzhashk",
    "wzhashkoons",
    "aanjtoon",
    "kiwenzii",
    "aanzkon'ye",
    "giigoonyke",
    "wmbaasin",
    "n'daǧshin",
    "gbaaǩnan",
    "n'zikweȟdizo",
    "benǧzheȟdizo",
    "n'shtigwaan",
    "gziibiigshtigwaanedog",
    "niishtna",
    "spogin",
    "shpiming"
]

# Iterate through the sample list of Nishnaabe words and split each
# Nishnaabe word into its respective constituent graphemes.
for nishsample in nishsamples:
    print(split_into_graphemes(nishsample))


from flask import g
import uuid
import hashlib

from expungeservice.database.db_util import rollback_errors
from expungeservice.models.expungement_result import EligibilityStatus
from flask_login import current_user

def save_result(request_data, record):

    user_id = current_user.user_id

    search_param_string = (
        user_id
        + request_data["first_name"]
        + request_data["last_name"]
        + request_data["middle_name"]
        + request_data["birth_date"]
    )

    hashed_search_params = hashlib.sha256(search_param_string.encode()).hexdigest()

    num_charges = len(record.charges)

    num_eligible_charges = len(
        [c for c in record.charges if c.expungement_result.type_eligibility.status == EligibilityStatus.ELIGIBLE]
    )

    _db_insert_result(g.database, user_id, hashed_search_params, num_charges, num_eligible_charges)

@rollback_errors
def _db_insert_result(database, user_id, hashed_search_params, num_charges, num_eligible_charges):

    database.cursor.execute(
        """
        INSERT INTO  SEARCH_RESULTS(search_result_id, user_id, hashed_search_params, num_charges, num_eligible_charges )
        VALUES ( uuid_generate_v4(), %(user_id)s, %(params)s, %(num_charges)s, %(num_eligible_charges)s);
        """,
        {
            "user_id": uuid.UUID(user_id).hex,
            "params": hashed_search_params,
            "num_charges": num_charges,
            "num_eligible_charges": num_eligible_charges,
        },
    )


counts = dict()

names = ['csev','cwen', 'csev', 'zqien', 'cwen']

for name in names:
  if name not in counts:
    counts[name] = 1
  else:
    counts[name] = counts[name] + 1
print(counts)

places = ['Miami', 'Tokyo', 'Paris', 'New York', 'Sao Polo', 'Miami', 'Paris', 'Tokyo']

for place in places:
  print(place)

# if places in counts:
#   x = counts[places]
# else:
#   x = 0
#   x = counts.get(places, 0)
# print(counts)