import numpy as np

from .newton_root import bisection, init_newton, newton, newton_s
from .utils import tensor_H

def _iss_single(s, X, W, r_inv):
    """
    Update of the demixing matrix with the iterative source steering of the s-th source
    """
    _, __, n_frames = X.shape

    # demix
    Y = W @ X

    # OP: n_frames * n_src
    v_num = (Y * r_inv[None, :, :]) @ np.conj(Y[:, s, :, None])  # (n_freq, n_src, 1)
    # OP: n_frames * n_src
    v_denom = r_inv[None, :, :] @ np.abs(Y[:, s, :, None]) ** 2
    # (n_freq, n_src, 1)

    # OP: n_src
    v = v_num[:, :, 0] / v_denom[:, :, 0]
    # OP: 1
    v[:, s] = 1.0 - np.sqrt(n_frames) / np.sqrt(v_denom[:, s, 0])

    # update demixed signals
    # OP: n_frames * n_src
    W[:] -= v[:, :, None] * W[:, None, s, :]

def _ip_single(s, X, W, r_inv):
    """
    Performs update of the s-th demixing vector using
    the iterative projection rules
    """

    n_freq, n_chan, n_frames = X.shape

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = np.matmul((X * r_inv[None, None, :]), tensor_H(X)) / n_frames

    WV = np.matmul(W, V)
    rhs = np.eye(n_chan)[None, :, s]  # s-th canonical basis vector
    W[:, s, :] = np.conj(np.linalg.solve(WV, rhs))

    # normalize
    denom = np.matmul(
        np.matmul(W[:, None, s, :], V[:, :, :]), np.conj(W[:, s, :, None])
    )
    W[:, s, :] /= np.sqrt(denom[:, :, 0])

def _ip_double(s1, s2, X, W, r_inv):
    """
    Performs a joint update of the s1-th and s2-th demixing vectors
    usint the iterative projection 2 rules
    """
    n_frames = X.shape[-1]

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = [
        (X * r_inv[None, i, None, :]) @ tensor_H(X) / n_frames
        for i, s in enumerate([s1, s2])
    ]

    _ip_double_sub(s1, s2, V, W)

def _ip_double_sub(s1, s2, V, W):

    n_freq, n_chan, _ = W.shape

    # right-hand side for computation of null space basis
    # no_update = [i for i in range(n_src) if i != s]
    rhs = np.eye(n_chan)[None, :, [s1, s2]]

    # Basis for demixing vector
    H = []
    HVH = []
    for i, s in enumerate([s1, s2]):
        H.append(np.linalg.solve(W @ V[i], rhs))
        HVH.append(tensor_H(H[i]) @ V[i] @ H[i])

    # Now solve the generalized eigenvalue problem
    lmbda_, R = np.linalg.eig(np.linalg.solve(HVH[0], HVH[1]))

    # Order by decreasing order of eigenvalues
    I_inv = lmbda_[:, 0] > lmbda_[:, 1]
    lmbda_[I_inv, :] = lmbda_[I_inv, ::-1]
    R[I_inv, :, :] = R[I_inv, :, ::-1]

    for i, s in enumerate([s1, s2]):
        denom = np.sqrt(np.conj(R[:, None, :, i]) @ HVH[i] @ R[:, :, i, None])
        W[:, s, None, :] = tensor_H(H[i] @ (R[:, :, i, None] / denom))

def _ip_double_two_channels(X, W, r_inv):
    """
    Specialized update rule for the 2 sources/2 channels case.
    In this, a globally optimal update of the surrogate function exists.
    """

    n_freq, n_chan, n_frames = X.shape
    s1, s2 = 0, 1

    assert n_chan == 2, "This update rule is only valid for two channels"

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = [
        (X * r_inv[None, i, None, :]) @ tensor_H(X) / n_frames
        for i, s in enumerate([s1, s2])
    ]

    # Now solve the generalized eigenvalue problem
    lmbda_, R = np.linalg.eig(np.linalg.solve(V[0], V[1]))

    # Order by decreasing order of eigenvalues
    I_inv = lmbda_[:, 0] < lmbda_[:, 1]
    lmbda_[I_inv, :] = lmbda_[I_inv, ::-1]
    R[I_inv, :, :] = R[I_inv, :, ::-1]

    for i, s in enumerate([s1, s2]):
        denom = np.sqrt(np.conj(R[:, None, :, i]) @ V[i] @ R[:, :, i, None])
        W[:, s, None, :] = tensor_H(R[:, :, i, None] / denom)

def _parametric_background_update(n_src, W, Cx):
    """
    Update the backgroud part of a parametrized demixing matrix
    """

    W_target = W[:, :n_src, :]  # target demixing matrix
    J = W[:, n_src:, :n_src]  # background demixing matrix

    tmp = np.matmul(W_target, Cx)
    J[:, :, :] = tensor_H(np.linalg.solve(tmp[:, :, :n_src], tmp[:, :, n_src:]))

def _joint_demix_background(s, n_src, X, W, r_inv, Cx):
    """
    Joint update of one demixing vector and one block
    """

    n_freq, n_chan, n_frames = X.shape

    # right-hand side for computation of null space basis
    # no_update = [i for i in range(n_src) if i != s]
    update = [s] + list(range(n_src, n_chan))
    rhs = np.eye(n_chan)[None, :, update]

    # Compute Auxiliary Variable
    # shape: (n_freq, n_chan, n_chan)
    V = (np.matmul((X * r_inv[None, None, :]), tensor_H(X))) / n_frames

    # Basis for demixing vector
    Hw = np.linalg.solve(W @ V, rhs)
    HVH = np.matmul(tensor_H(Hw), V @ Hw)

    # Basis for the background
    Hb = np.linalg.solve(W @ Cx, rhs)
    HCH = np.matmul(tensor_H(Hb), Cx @ Hb)

    # Now solve the generalized eigenvalue problem
    B_H = np.linalg.cholesky(HCH)
    B_inv = np.linalg.inv(tensor_H(B_H))
    V_tilde = np.linalg.solve(B_H, HVH) @ B_inv
    lmbda_, R = np.linalg.eigh(V_tilde)
    R = np.matmul(B_inv, R)

    # The target demixing vector requires normalization
    R[:, :, -1] /= np.sqrt(lmbda_[:, -1, None])

    # Assign to target and background
    W[:, s, None, :] = tensor_H(Hw @ R[:, :, -1, None])
    W[:, n_src:, :] = tensor_H(Hb @ R[:, :, :-1])

def _block_ip(sources, X, W, V):
    """
    Block iterative projection update
    This is a joint update of the demixing vectors of sources
    that are not independent
    """

    n_freq, n_chan, n_frames = X.shape

    rhs = np.eye(n_chan)[None, :, sources]
    W[:, sources, :] = tensor_H(np.linalg.solve(W @ V, rhs))

    for s in sources:
        # normalize
        denom = np.matmul(
            np.matmul(W[:, None, s, :], V[:, :, :]), np.conj(W[:, s, :, None])
        )
        W[:, s, :] /= np.sqrt(denom[:, :, 0])

def _ipa_make_T(u, q, k):

    n_freq, n_chan = u.shape

    o = list(range(n_chan))
    o.remove(k)

    T = np.zeros((n_freq, n_chan, n_chan), dtype=u.dtype)
    T[:] = np.eye(n_chan)[None, :, :]
    T[:, o, k] = np.conj(q)
    T[:, k, :] = np.conj(u)

    return T

def _ipa(V, W, k):
    """
    Implementation of Iterative Projection with Adjustment (IPS) updates for IVA using the global solution based
    on Newton-Raphson root finding
    """
    n_freq, n_chan, _ = W.shape

    o = list(range(n_chan))
    o.remove(k)

    Vk = W @ V[k] @ tensor_H(W)
    Vk_inv = np.linalg.inv(Vk)

    # shape (n_freq, n_chan - 1, 1)
    a = np.real(
        [(W[:, None, k, :] @ V[m] @ np.conj(W[:, k, :, None]))[..., 0] for m in o]
    ).transpose([1, 0, 2])
    a_inv_sq = 1.0 / np.sqrt(a)

    # shape (n_freq, n_chan - 1, 1)
    b = np.array(
        [(W[:, None, k, :] @ V[m] @ np.conj(W[:, m, :, None]))[..., 0] for m in o]
    ).transpose([1, 0, 2])

    u, q = _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b)

    T = _ipa_make_T(u[:, :, 0], q[:, :, 0], k)
    W = T @ W

    return W

def _ipa2(k, Y, W, r_inv):

    n_freq, n_chan, n_frames = Y.shape

    o = list(range(n_chan))
    o.remove(k)

    Vk = (Y * r_inv[k, None, None, :]) @ tensor_H(Y) / n_frames
    Vk = 0.5 * (Vk + np.conj(Vk))
    Vk_inv = np.linalg.inv(Vk)

    # shape (n_freq, n_chan - 1, 1)
    a = (r_inv[None, o, :]) @ np.abs(Y[:, k, :, None]) ** 2
    a /= n_frames
    a_inv_sq = 1.0 / np.sqrt(a)

    # shape (n_freq, n_chan - 1, 1)
    b = (Y[:, o, :] * r_inv[None, o, :]) @ np.conj(Y[:, k, :, None])
    b = np.conj(b)
    b /= n_frames

    u, q = _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b)

    # compute the new separation matrix
    T = _ipa_make_T(u[:, :, 0], q[:, :, 0], k)
    W[:] = T @ W

    # update the output signal
    Yk = tensor_H(u) @ Y
    for i, ell in enumerate(o):
        Y[:, ell, :] += np.conj(q[:, i, :]) * Y[:, k, :]
    Y[:, [k], :] = Yk

def _ipa_sub(k, Vk, Vk_inv, a, a_inv_sq, b):

    n_freq, n_chan, _ = Vk.shape
    o = list(range(n_chan))
    o.remove(k)

    # shape (n_freq, n_chan, n_chan)
    C = np.conj(Vk_inv[:, :, o][:, o, :])

    # shape (n_freq, n_chan, n_chan)
    U = a_inv_sq * C * tensor_H(a_inv_sq)
    # sometimes, the hermitian symmetry is lost due to small numerical innacuracies
    # and numpy.linalg.eigh returns negative eigenvalues, throwing off the algorithm,
    # we thus enforce hermtian symmetry of U here
    U = 0.5 * (U + tensor_H(U))

    # shapes (n_freq, n_chan), (n_freq, n_chan - 1, n_chan - 1)
    phi, Sigma = np.linalg.eigh(U)

    # shape (n_freq, n_chan - 1, 1)
    ag = a_inv_sq * np.conj(Vk_inv[:, o, k, None])

    # shape (n_freq)
    g_C_inv_g = np.sum(np.abs(tensor_H(Sigma) @ ag) ** 2 / phi[:, :, None], axis=(1, 2))
    z = np.real(Vk_inv[:, k, k] - g_C_inv_g)

    # shape (n_freq, n_chan, 1)
    v = tensor_H(Sigma) @ (-U @ (a_inv_sq * b) - ag)

    # normalize so that the largest eigenvalue is one
    phi_max = phi[:, -1]
    phi_hat = phi / phi_max[:, None]
    v_hat = v[:, :, 0] / phi_max[:, None]
    z_hat = z / phi_max

    # make an array to receive the solution
    q = np.zeros((n_freq, n_chan - 1, 1), dtype=Vk.dtype)
    lambda_ = np.zeros(n_freq, dtype=np.float)

    # when v is very small, the solution is given by the dominant eigenvector
    # of U
    I_small = np.linalg.norm(v[:, :, 0], axis=1) < 1e-10
    if np.sum(I_small) > 0:

        # there is one more case we need to account for
        # when z is larger than the largest eigenvalue,
        # the optimal q vector is zero, and lambda = z
        I_z = np.logical_and(I_small, z >= phi_max)
        lambda_[I_z] = z[I_z]

        # when z is smaller than the largest eigenvalue
        # then lambda = largest eigenvalue and q is the
        # corresponding eigenvector
        I_ow = np.logical_and(I_small, z < phi_max)
        uv = Sigma[I_ow, :, -1, None]  # leading eigenvector
        lambda_[I_ow] = phi[I_ow, -1]

        scale = np.sqrt(phi[I_ow, -1] - z[I_ow]) / np.sqrt(
            np.real(tensor_H(uv) @ U[I_ow] @ uv)[:, 0, 0]
        )
        q[I_ow, :, 0] = (
            a_inv_sq[I_ow, :, 0] * (uv[:, :, 0] * scale[:, None]) + b[I_ow, :, 0]
        )

    # initial value for root finding
    # anything larger than the largest eigenvalue should be fine
    I_big = np.logical_not(I_small)

    # use the smart initialization
    lambda_0 = init_newton(phi_hat[I_big, :], v_hat[I_big, :], z_hat[I_big])

    # newton
    lambda_[I_big] = newton(
        np.maximum(lambda_0, z_hat[I_big]),
        phi_hat[I_big, :],
        v_hat[I_big, :],
        z_hat[I_big],
        atol=1e-5,
        max_iter=1000,
        verbose=False,
        plot=False,
    )

    # rescale
    lambda_[I_big] *= phi_max[I_big]

    # finally piece together the optimum vector
    t = v[I_big, :] / (lambda_[I_big, None, None] - phi[I_big, :, None])
    q[I_big, :] = a_inv_sq[I_big] * (Sigma[I_big] @ t) - b[I_big] / a[I_big]

    # build the other demixing vector
    x = np.ones((n_freq, n_chan, 1), dtype=Vk.dtype)
    x[:, o, :] = -np.conj(q)
    u = Vk_inv[:, :, k, None] - Vk_inv[:, :, o] @ np.conj(q)

    I = np.where(lambda_ <= 0.0)[0]
    if len(I) > 0:
        print("Detected bad cases, investigate:")
        v_not_hat = -U @ (a_inv_sq * b) - ag
        for i in I:
            print("*****")
            print("freq:", i)
            print("U=")
            print(U[i])
            print("v=")
            print(v_not_hat[i])
            print("z=", z[i])
            print("phi=")
            print(phi[i])
            print("v_hat=")
            print(v[i])
        print("*****")
        raise ValueError(
            "Numerical problem. Please report to "
            " https://github.com/fakufaku/auxiva-ipa"
        )

    u *= 1.0 / np.sqrt(lambda_[:, None, None])

    return u, q


import datetime

def int_to_time(int_number):
    temptime = datetime.timedelta(milliseconds=int_number)
    minutes = temptime.seconds // 60
    seconds = temptime.seconds % 60
    milliseconds = round(temptime.microseconds / 1000)
    if milliseconds < 100:
        milliseconds = milliseconds / 1000
        if seconds < 10:
            return str(minutes) + ":0" + str(seconds + milliseconds)
        else:
            return str(minutes) + ":" + str(seconds + milliseconds)
    else:
        if seconds < 10:
            return str(minutes) + ":0" + str(seconds) + "." + str(milliseconds)
        else:
            return str(minutes) + ":" + str(seconds) + "." + str(milliseconds)


"""
Using Asyncio and AioHTTP to improve IO HTTP requests to Console Server and Azure Devops

Need to figure out the syntax of the Asyncio and AIOHTTP library, could improve by 5 to 6 times for performance.

There are two types of IO bound operations that we need to think about when dealing with Asynchronous programming.

One being the file system and other being the network, particularly using the requests module.

Typically speaking, we have 2 types of operations. Normally we write code that is synchronous. In this fashion, we
write code that waits until a sequence of executions is done before moving onto the next operations.  That's fine and
is very typical in most programming applications.

However, there will be a time when data needs to fetched with millions of requests that could take hours for a computer
to gather.  For this type of operation, we need to improve the performance and we could accomplish this with
non-blocking expressions that would help improve performance.
"""

import asyncio
import json

import aiohttp
from aiohttp import client_exceptions
from colorama import Fore, Style

from pipe_cleaner.src.data_access import write_host_groups_json, get_all_host_ids, get_dhcp

# from data_access import write_host_groups_json, get_all_host_ids, get_dhcp

# Store all tickets in a particular pipe. Clears after each new pipe
current_pipe_tickets: list = []
all_tickets: list = []
all_vm_data: dict = {}
total_systems_in_pipe: list = []
total_tickets_in_pipe: list = []
total_vms: list = []
total_systems: list = []

virtual_machine_data: list = []

# Stores the console server here, returns after collecting all data
console_server_data: dict = {}

# Stores Overall Workload of each VSE employee based on 'checked out to' in Console Server
overall_workload: dict = {}

# Commodity Serial Numbers
all_serial: list = []

# Commodity Part Numbers
all_part_numbers: list = []

async def generate_console_server_json(client: asyncio, host_id: str) -> str:
    """
    Generates the JSON data from the Host Details page using the Host ID.
    Returns product-serial if JSON data is generated properly.

    :param client:
    :param host_id: found on Host Details page in URL ?host_id=<some_id>get_system_json
    :return: returns product-serial string for getting JSON data
    """
    console_server_url = 'http://172.30.1.100/console/console_js.php'
    generate_data = {
        'action': 'get_json_data',
        'host_id': f'{host_id}'
    }
    host_name_data = {
        'action': 'get_host_name_data',
        'host_id': f'{host_id}'
    }
    async with client.post(url=console_server_url, json=generate_data) as response_generate:
        if response_generate.status == 200:
            async with client.post(url=console_server_url, json=host_name_data) as response_product_serial:
                assert response_product_serial.status == 200
                response_text = await response_product_serial.text()

                try:
                    product = json.loads(str(response_text))['host_name_data']['product']
                    serial = json.loads(str(response_text))['host_name_data']['serial']
                    return f'{product}-{serial}'
                except json.decoder.JSONDecodeError:
                    pass

async def get_system_json(client, product_serial: str, host_id: str) -> dict:
    """
    Gets the Generated data using the product_serial string and creates JSON file.
    Returns JSON of Host within Console Server.

    :param client:
    :param product_serial: string from generate_json_data method
    :param host_id: 24-character string host id in URL of Host Details in Console Server
    :return: JSON data of Host
    """
    data = {
        'action': 'get_json_data',
        'host_id': f'{host_id}'
    }
    async with client.post(url=f'http://172.30.1.100/results/{product_serial}.json', json=data) as response_data:
        # assert response_data.status == 200

        data = await response_data.text()
        try:
            return json.loads(data)
        except json.decoder.JSONDecodeError:
            pass

def calculate_last_time_alive(raw_last_found_alive: str) -> float:
    """
    Calculate last time alive in non-seconds. Raw last time alive is in seconds and ain't no body got time for that.
    :param raw_last_found_alive:
    :return:
    """
    import time
    return float(time.time()) - float(raw_last_found_alive)

def check_none_type(component):
    """
    Checks if NoneType exception occurred
    :param component:
    :return:
    """
    if component is None:
        return 'None'
    else:
        return component

def get_virtual_machine_data(system_json: dict, pipe_name: str) -> None:
    """
    Get virtual machine information for engineers to lower confusion on VM usage
    :param pipe_name: host group name
    :param system_json: system data based from Console Server
    :return: appends to list for later gathering in the Console Server data
    """
    # check_none_type function accounts for rare case of NoneType errors
    vm_data: dict = {'pipe_name': check_none_type(pipe_name),
                     'machine_name': check_none_type(system_json.get('machine_name')),
                     'host_id': check_none_type(system_json.get('id')),
                     'connection_status': check_none_type(system_json.get('connection_status')),
                     'last_found_alive': check_none_type(system_json.get('last_found_alive')),
                     'location': check_none_type(system_json.get('location')),
                     'comment': check_none_type(system_json.get('comment')),
                     'ssh_connection_string': check_none_type(system_json.get('ssh_connection_string')),
                     'rdp_connection_string': check_none_type(system_json.get('rdp_connection_string')),
                     'vnc_connection_string': check_none_type(system_json.get('vnc_connection_string')),
                     'host_ip': check_none_type(system_json.get('net', {}).get('interfaces', {})[0].get('ip')),
                     'checked_out_to': check_none_type(system_json.get('checked_out_to'))}

    virtual_machine_data.append(vm_data)

def check_username_is_empty(username: str) -> bool:
    """

    :param username: Username with underscore separator
    :return: True or False
    """
    if not username or username.upper() == 'NONE':
        return False
    else:
        return True

def check_username_in_userbase(user_name: str) -> bool:
    """
    Ensures the username is already in the dictionary

    :param user_name: Individual
    :return: True or False
    """
    user_base: dict = console_server_data['user_base']

    if check_username_is_empty(user_name) is True and user_name in user_base:
        return True
    else:
        return False

def count_username(clean_username: str) -> None:
    """

    :param clean_username:
    """
    check_user_base: bool = check_username_in_userbase(clean_username)
    is_username_empty: bool = check_username_is_empty(clean_username)

    if is_username_empty is False:
        pass

    elif check_user_base is False:
        console_server_data['user_base'][clean_username]: dict = {}
        console_server_data['user_base'][clean_username]['count']: int = 1
        console_server_data['user_base'][clean_username]['default_name']: str = clean_username.replace('_', '.')
        console_server_data['user_base'][clean_username]['alt_name']: str = clean_username.replace('_', ' '). \
            title().replace(' ', '')
        console_server_data['user_base'][clean_username]['systems']: dict = {}
        console_server_data['user_base'][clean_username]['virtual_machines']: dict = {}

    elif check_user_base is True:
        console_server_data['user_base'][clean_username]['count'] += 1

def store_machine_name(user_name: str, machine_name: str) -> None:
    """

    :param machine_name:
    :param user_name:
    """
    is_username_empty: bool = check_username_is_empty(user_name)

    if is_username_empty is False:
        pass

    elif machine_name not in console_server_data['user_base'][user_name]['systems']:
        if '-VM-' in machine_name:
            console_server_data['user_base'][user_name]['virtual_machines'][machine_name]: dict = {}

        else:
            console_server_data['user_base'][user_name]['systems'][machine_name]: dict = {}

def store_pipe_name(user_name: str, machine_name: str, pipe_name: str):
    """

    :param user_name:
    :param machine_name:
    :param pipe_name:
    :return:
    """
    is_username_empty: bool = check_username_is_empty(user_name)

    if is_username_empty is False:
        pass

    else:
        if '-VM-' in machine_name:
            console_server_data['user_base'][user_name]['virtual_machines'][machine_name]['pipe_name'] = pipe_name

        else:
            console_server_data['user_base'][user_name]['systems'][machine_name]['pipe_name'] = pipe_name

def process_pipe_name(pipe_name: str):
    """
    Shorten pipe name to fit into excel output
    :param pipe_name:
    :return:
    """
    clean_data: str = pipe_name. \
        replace('[', ''). \
        replace(']', ''). \
        replace("'", '')

    last_part: str = clean_data.split(' ')[-1]

    return clean_data.replace('Pipe-', '').replace(last_part, '').strip()

def store_system_data(system_json: dict, pipe_name: str):
    """
    Grab relevant information based on needs. Stores into Console Server data structure
    :param pipe_name:
    :param system_json:
    """
    # Accounts for a system
    total_systems.append(1)
    # Temporary Data Structure to hold System Data
    try:
        current_system_data: dict = {'machine_name': str(system_json.
                                                         get('machine_name', 'None'))}
        current_machine_name: str = system_json.get('machine_name')

        if 'VSE' in current_machine_name and '-VM-' in current_machine_name:
            get_virtual_machine_data(system_json, pipe_name)

        if 'VSE' in current_machine_name and '-VM-' not in current_machine_name:
            setup_data: list = console_server_data[pipe_name]['setup_data']['total_systems']
            setup_data.append(1)

        current_system_data['ticket']: dict = str(system_json.get('ticket', 'None'))

        # Stores all tickets for unique tickets data and pipe-specific tickets
        ticket = current_system_data.get('ticket', 'None')
        if ticket != '' and ticket.isdigit() is True and '-VM-' not in current_system_data.get('machine_name'):
            all_tickets.append(ticket)
            current_pipe_tickets.append(ticket)
            setup_data: list = console_server_data[pipe_name]['setup_data']['systems_with_ticket']
            setup_data.append(1)

        # Username
        checked_out_to = str(system_json.get('checked_out_to'))
        current_system_data['checked_out_to']: dict = checked_out_to

        # Upper for consistent comparison
        upper_machine_name = str(current_system_data.get('machine_name', 'None'))

        connection_status = str(system_json.get('connection_status', 'None'))

        # Stores checked_out_to for pipe-level work
        clean_username: str = checked_out_to.lower().replace('.', '_')

        if clean_username not in overall_workload:
            overall_workload[clean_username] = {}

        if 'systems' not in overall_workload[clean_username]:
            overall_workload[clean_username]['systems'] = 0

        if 'systems' in overall_workload[clean_username]:
            overall_workload[clean_username]['systems'] += 1

        # Last Time Alive
        raw_last_found_alive = str(system_json.get('last_found_alive', 'None'))
        last_alive: float = calculate_last_time_alive(raw_last_found_alive)
        current_system_data['last_found_alive']: float = last_alive

        # Machine Serial Number
        machine_serial_number: str = system_json.get('bmc', {}).get('fru', {}).get('product', {}).get('serial')

        machine_product_name: str = system_json.get('bmc', {}).get('fru', {}).get('product', {}).get('name')

        # Comment / Status
        current_system_data['comment']: dict = str(system_json.
                                                   get('comment', 'None'))

        # Status ex. Idle, Offline
        current_system_data['system_status']: dict = str(system_json.
                                                         get('status', 'None'))

        # Connection Status
        current_system_data['connection_status']: dict = str(system_json.
                                                             get('connection_status', 'None'))

        # Location ex. R44U14N14
        current_system_data['location']: dict = str(system_json.
                                                    get('location', 'None'))

        host_ip: str = str(system_json['net']['interfaces'][0].get('ip', 'None'))
        current_system_data['host_ip']: dict = host_ip

        # BIOS ex. C2030.BS.2A42.AF1
        current_system_data['server_bios']: dict = str(system_json.
                                                       get('dmi', {}).
                                                       get('bios', {}).
                                                       get('version', 'None'))

        # Processor ex. Intel or AMD
        current_system_data['processors']: dict = str(system_json.
                                                      get('dmi', {}).
                                                      get('procs', 'None'))

        # BMC ex. 3.30
        current_system_data['server_bmc']: dict = str(system_json.
                                                      get('bmc', {}).
                                                      get('mc', {}).
                                                      get('firmware', 'None'))

        # CPLD Version ex. 000000014
        current_system_data['server_cpld']: dict = str(system_json.
                                                       get('cpld', {}).
                                                       get('secure_cpld_version', 'None'))

        # CPLD Sequence Version ex. 00000014
        current_system_data['sequence_cpld_version']: dict = str(system_json.
                                                                 get('cpld', {}).
                                                                 get('sequence_cpld_version', 'None'))

        # Platform Version
        current_system_data['server_os']: dict = str(system_json.
                                                     get('platform', {}).
                                                     get('version', 'None'))

        # TPM
        current_system_data['server_tpm']: dict = str(system_json.
                                                      get('tpm', {}).
                                                      get('version', 'None'))

        # Host ID ex. 5e5460a20b4b023d327dda51
        current_system_data['id']: dict = str(system_json.
                                              get('id', 'None'))

        # DIMMs ex.
        current_system_data['system_dimms']: dict = str(system_json.
                                                        get('dmi', {}).
                                                        get('dimms', 'None'))

        # Unique DIMMs
        unique_dimms: dict = system_json.get('dmi', {}).get('unique_dimms', 'None')
        for unique_dimm in unique_dimms:
            part_number: list = unique_dimm.get('part')
            count = int(unique_dimm.get('count'))

            clean_pipe_name: str = process_pipe_name(pipe_name)
            part_numbers: dict = console_server_data['part_numbers']

            if part_number in part_numbers:
                part_numbers[part_number]['quantity'] += count
                locations: dict = part_numbers[part_number]['locations']

                if clean_pipe_name in locations:
                    locations[clean_pipe_name]['count'] += count

                else:
                    locations[clean_pipe_name]: dict = {}
                    locations[clean_pipe_name]['count'] = count
                    locations[clean_pipe_name]['connection']: int = connection_status
                    locations[clean_pipe_name]['last_alive']: float = last_alive

            else:
                part_numbers[part_number]: dict = {}
                part_numbers[part_number]['locations']: dict = {}
                part_numbers[part_number]['locations'][clean_pipe_name]: dict = {}

                part_numbers[part_number]['quantity']: int = count
                part_numbers[part_number]['locations'][clean_pipe_name]['count']: int = count
                part_numbers[part_number]['locations'][clean_pipe_name]['connection']: int = connection_status
                part_numbers[part_number]['locations'][clean_pipe_name]['last_alive']: float = last_alive

            if not part_number:
                pass

            else:
                if part_number not in console_server_data['inventory']['commodities']['dimms']:
                    console_server_data['inventory']['commodities']['dimms'][part_number] = count
                else:
                    console_server_data['inventory']['commodities']['dimms'][part_number] += count

                if part_number not in console_server_data[pipe_name]['pipe_data']['pipe_inventory']['dimms']:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['dimms'][part_number] = count
                else:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['dimms'][part_number] += count

                try:
                    current_commodity: dict = {'pipe_name': pipe_name,
                                               'connection_status': connection_status,
                                               'last_found_alive': current_system_data['last_found_alive'],
                                               'machine_unique_id': f'{machine_product_name}-'
                                                                    f'{machine_serial_number}',
                                               'machine_sku': machine_product_name,
                                               'machine_serial': machine_serial_number,
                                               'supplier': unique_dimm['manufacturer'],
                                               'machine_name': upper_machine_name,
                                               'commodity_type': 'NVMe',
                                               'part_number': part_number,
                                               'count': count,
                                               'type': 'DIMM'}

                    all_part_numbers.append(current_commodity)
                except KeyError:
                    pass

        all_dimms: dict = system_json.get('dmi', {}).get('dimms')
        if not all_dimms:
            pass

        else:
            for current_dimm in all_dimms:
                if not current_dimm:
                    pass
                else:
                    try:
                        current_commodity: dict = {'pipe_name': pipe_name,
                                                   'connection_status': connection_status,
                                                   'machine_unique_id': f'{machine_product_name}-'
                                                                        f'{machine_serial_number}',
                                                   'machine_sku': machine_product_name,
                                                   'machine_serial': machine_serial_number,
                                                   'machine_name': upper_machine_name,
                                                   'commodity_type': 'DIMM',
                                                   'part_number': current_dimm['part'],
                                                   'serial_number': current_dimm['serial']}
                        all_serial.append(current_commodity)
                    except KeyError:
                        pass

        current_system_data['unique_dimms']: dict = unique_dimms

        # NVMes
        current_system_data['system_nvmes']: dict = str(system_json.
                                                        get('nvme', {}).
                                                        get('nvmes', 'None'))

        # Unique NVMes
        unique_nvmes: dict = system_json.get('nvme', {}).get('unique_nvmes')
        for unique_nvme in unique_nvmes:
            nvme_model = get_nvme_model(unique_nvme)
            count = int(unique_nvme.get('count'))

            clean_pipe_name: str = process_pipe_name(pipe_name)
            part_numbers: dict = console_server_data['part_numbers']

            if nvme_model in part_numbers:
                part_numbers[nvme_model]['quantity'] += count
                locations: dict = part_numbers[nvme_model]['locations']

                if clean_pipe_name in locations:
                    locations[clean_pipe_name]['count'] += count

                else:
                    locations[clean_pipe_name]: dict = {}
                    locations[clean_pipe_name]['count'] = count
                    locations[clean_pipe_name]['connection']: int = connection_status
                    locations[clean_pipe_name]['last_alive']: float = last_alive

            else:
                part_numbers[nvme_model]: dict = {}
                part_numbers[nvme_model]['locations']: dict = {}
                part_numbers[nvme_model]['locations'][clean_pipe_name]: dict = {}

                part_numbers[nvme_model]['quantity']: int = count
                part_numbers[nvme_model]['locations'][clean_pipe_name]['count']: int = count
                part_numbers[nvme_model]['locations'][clean_pipe_name]['connection']: int = connection_status
                part_numbers[nvme_model]['locations'][clean_pipe_name]['last_alive']: float = last_alive

            if not nvme_model:
                pass
            else:
                if nvme_model not in console_server_data['inventory']['commodities']['nvmes']:
                    console_server_data['inventory']['commodities']['nvmes'][nvme_model] = count
                else:
                    console_server_data['inventory']['commodities']['nvmes'][nvme_model] += count

                if nvme_model not in console_server_data[pipe_name]['pipe_data']['pipe_inventory']['nvmes']:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['nvmes'][nvme_model] = count
                else:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['nvmes'][nvme_model] += count

                try:
                    current_commodity: dict = {'pipe_name': pipe_name,
                                               'connection_status': connection_status,
                                               'last_found_alive': current_system_data['last_found_alive'],
                                               'machine_unique_id': f'{machine_product_name}-'
                                                                    f'{machine_serial_number}',
                                               'machine_sku': machine_product_name,
                                               'machine_serial': machine_serial_number,
                                               'machine_name': upper_machine_name,
                                               'commodity_type': 'NVMe',
                                               'part_number': nvme_model,
                                               'count': count,
                                               'type': 'NVMe'}

                    all_part_numbers.append(current_commodity)
                except KeyError:
                    pass

        current_system_data['unique_nvmes']: dict = unique_nvmes

        all_nvmes: dict = system_json.get('nvme', {}).get('nvmes')
        if not all_nvmes:
            pass

        else:
            for current_nvme in all_nvmes:

                if not current_nvme:
                    pass

                else:
                    try:
                        current_commodity: dict = {'pipe_name': pipe_name,
                                                   'connection_status': connection_status,
                                                   'machine_unique_id': f'{machine_product_name}-'
                                                                        f'{machine_serial_number}',
                                                   'machine_sku': machine_product_name,
                                                   'machine_serial': machine_serial_number,
                                                   'machine_name': upper_machine_name,
                                                   'commodity_type': 'NVMe',
                                                   'part_number': current_nvme['model'],
                                                   'serial_number': current_nvme['serial']}

                        all_serial.append(current_commodity)
                    except KeyError:
                        pass

        # Disks
        current_system_data['system_disks']: dict = str(system_json.
                                                        get('disk', {}).
                                                        get('disks', 'None'))

        # Unique Disks
        unique_disks: dict = system_json.get('disk', {}).get('unique_disks', 'None')

        for unique_disk in unique_disks:
            disk_part_number: str = get_disk_part_number(unique_disk)
            count = int(unique_disk.get('count'))

            clean_pipe_name: str = process_pipe_name(pipe_name)
            part_numbers: dict = console_server_data['part_numbers']

            if disk_part_number in part_numbers:
                part_numbers[disk_part_number]['quantity'] += count
                locations: dict = part_numbers[disk_part_number]['locations']

                if clean_pipe_name in locations:
                    locations[clean_pipe_name]['count'] += count

                else:
                    locations[clean_pipe_name]: dict = {}
                    locations[clean_pipe_name]['count'] = count
                    locations[clean_pipe_name]['connection']: int = connection_status
                    locations[clean_pipe_name]['last_alive']: float = last_alive

            else:
                part_numbers[disk_part_number]: dict = {}
                part_numbers[disk_part_number]['locations']: dict = {}
                part_numbers[disk_part_number]['locations'][clean_pipe_name]: dict = {}

                part_numbers[disk_part_number]['quantity']: int = count
                part_numbers[disk_part_number]['locations'][clean_pipe_name]['count']: int = count
                part_numbers[disk_part_number]['locations'][clean_pipe_name]['connection']: int = connection_status
                part_numbers[disk_part_number]['locations'][clean_pipe_name]['last_alive']: float = last_alive

            if not disk_part_number or disk_part_number == 'Virtual HD':
                pass

            else:
                if disk_part_number not in console_server_data['inventory']['commodities']['disks']:
                    console_server_data['inventory']['commodities']['disks'][disk_part_number] = count
                else:
                    console_server_data['inventory']['commodities']['disks'][disk_part_number] += count

                if disk_part_number not in console_server_data[pipe_name]['pipe_data']['pipe_inventory']['disks']:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['disks'][disk_part_number] = count
                else:
                    console_server_data[pipe_name]['pipe_data']['pipe_inventory']['disks'][disk_part_number] += count

                try:
                    current_commodity: dict = {'pipe_name': pipe_name,
                                               'connection_status': connection_status,
                                               'last_found_alive': current_system_data['last_found_alive'],
                                               'machine_unique_id': f'{machine_product_name}-'
                                                                    f'{machine_serial_number}',
                                               'machine_sku': machine_product_name,
                                               'machine_serial': machine_serial_number,
                                               'machine_name': upper_machine_name,
                                               'commodity_type': 'NVMe',
                                               'part_number': disk_part_number,
                                               'count': count,
                                               'type': 'Disk'}

                    all_part_numbers.append(current_commodity)
                except KeyError:
                    pass

        current_system_data['unique_disks']: dict = unique_disks

        all_disks: dict = system_json.get('disk', {}).get('disks')
        if not all_disks:
            pass

        else:
            for current_disk in all_disks:
                if not current_disk:
                    pass

                else:
                    try:
                        current_commodity: dict = {'pipe_name': pipe_name,
                                                   'connection_status': connection_status,
                                                   'machine_unique_id': f'{machine_product_name}-'
                                                                        f'{machine_serial_number}',
                                                   'machine_sku': machine_product_name,
                                                   'machine_serial': machine_serial_number,
                                                   'machine_name': upper_machine_name,
                                                   'commodity_type': 'Disk',
                                                   'part_number': current_disk['model'],
                                                   'serial_number': current_disk['serial']}
                        all_serial.append(current_commodity)

                    except KeyError:
                        pass

        count_username(clean_username)
        store_machine_name(clean_username, upper_machine_name)
        store_pipe_name(clean_username, upper_machine_name, pipe_name)

        # Avoid other
        if 'VSE' not in upper_machine_name and '-VM-' not in upper_machine_name:
            pass

        # Tallies total VMs and stores VM data
        elif '-VM-' in upper_machine_name:
            all_vm_data[upper_machine_name]: dict = {}
            all_vm_data[upper_machine_name]['name'] = upper_machine_name
            all_vm_data[upper_machine_name]['username']: dict = str(system_json.get('username', 'None'))
            all_vm_data[upper_machine_name]['host_ip']: dict = str(system_json.get('id', 'None'))
            total_vms.append(1)

        # Actually now stores data inside pipe information, CMA accounts for Chassis Manager Assembly, usually GEN 5
        elif 'VSE' in upper_machine_name and 'CMA-' not in upper_machine_name and '-VM-' not in upper_machine_name:

            # Accounts for setup information later in Setup Dashboard
            if not current_system_data.get('ticket') and str(current_system_data.get('ticket')).isdigit():
                total_tickets_in_pipe.append(1)
            total_systems_in_pipe.append(1)
            console_server_data[pipe_name]['pipe_data'][upper_machine_name]: dict = current_system_data

    except AttributeError:
        pass

def get_disk_part_number(unique_disk):
    split_parts: list = unique_disk.get('model').replace('  ', ' ').replace('_', ' ').split(' ')

    if len(split_parts) >= 3:
        for item in split_parts:
            if 'Micron' in item:
                return split_parts[-1].strip()
        else:
            return unique_disk.get("model").strip()

    count: int = 0
    for part_split in split_parts:
        if part_split.isalpha():
            count += 1

    if count == len(split_parts):
        return unique_disk.get('model').strip()

    else:
        for part_split in split_parts:
            if not part_split.isalpha():
                if not part_split:
                    return part_split
                else:
                    return part_split
        else:
            return unique_disk.get('model')

def get_nvme_model(unique_nvme):
    for part_split in unique_nvme.get('model').split(' '):
        if not part_split.isalpha():
            if not part_split:
                return part_split
            else:
                return part_split
    else:
        return ''

async def generate_json(client: asyncio, host_id: str, pipe_name: str):
    """
    Get JSON data from Console Server then stores data from
    :param client: async Session
    :param host_id:
    :param pipe_name:
    :return:
    """
    product_serial: str = await generate_console_server_json(client, host_id)
    try:
        upper_product_serial: str = product_serial.upper()

        if 'NONE' not in upper_product_serial:
            system_json: dict = await get_system_json(client, product_serial, host_id)

            store_system_data(system_json, pipe_name)

    except AttributeError:
        pass

async def start_loop(host_ids: list, pipe_name: str):
    """
    Generate Data, fetch data, and process data into a dictionary to analyze later
    :return:
    """
    event_loop = asyncio.get_running_loop()
    async with aiohttp.ClientSession(loop=event_loop) as client:
        tasks = [generate_json(client, host_id, pipe_name) for host_id in host_ids]
        await asyncio.gather(*tasks)

def get_group_unique_tickets(host_id: str, pipe_name: str) -> list:
    """
    Get the Host Group unique tickets for later extraction
    Note: THis is for all host
    :param pipe_name:
    :param host_id: Host Group ID
    :return:
    """
    # Store data
    host_group_unique_tickets: list = []

    # Single pipe data in JSON
    host_group_json: list = get_all_host_ids(host_id, pipe_name)

    for system in host_group_json:
        ticket: str = str(system['ticket'])

        # TRR IDs should only be numbers only
        if ticket.isdigit() is True:
            host_group_unique_tickets.append(ticket)

    # Insures only unique tickets
    return list(set(host_group_unique_tickets))

def get_overall_workload():
    return overall_workload

def get_total_systems():
    return sum(total_systems)

def main_method() -> dict:
    """
    Fetches all relevant information from Console Server using Async approach, accounts for Setup dashboard as well
    :return: returns dictionary showing for all pipe information first, note the first layer of the dictionary structure
    includes the unique tickets and will shoot an error message if not accounted for with 'Pipe-'
    """
    console_server_data['user_base']: dict = {}

    # Gets data and stores in JSON file for later use
    # Don't need async for this considering it's just one request
    json_file: dict = write_host_groups_json()
    host_groups_page: dict = json_file['host_groups']

    # Logs skipped or collected data based on naming of Host Groups
    log_total: list = []
    log_skipped: list = []
    log_collect: list = []
    log_non_pipes: list = []
    log_off_lines: list = []
    log_idle: list = []

    console_server_data['inventory']: dict = {}
    console_server_data['inventory']['commodities']: dict = {}
    console_server_data['inventory']['commodities']['dimms']: dict = {}
    console_server_data['inventory']['commodities']['nvmes']: dict = {}
    console_server_data['inventory']['commodities']['disks']: dict = {}

    console_server_data['part_numbers']: dict = {}

    # Tells user progress in Terminal environment
    print(f'\n\t=====================================================================')
    print(f'\t  Console Server - Collecting and Processing Data:')
    print(f'\t=====================================================================')
    print(f'\t\t  STATUS   |  REASON    |  HOST GROUP NAME')

    # Note that Host Groups is synonymous with Pipes
    for host_group in host_groups_page:

        # Gather basic Pipe information as Host Group, converts to upper and string to ensure proper comparison
        host_group_name: str = host_group.get('name')
        description: str = host_group.get('description')
        host_id: str = str(host_group.get('id')).upper()
        host_ids: list = host_group.get('host_ids')
        status: str = str(host_group.get('comment')).upper()
        checked_out_to: str = str(host_group.get('checked_out_to')).upper()

        # Stores checked_out_to for pipe-level work
        clean_username = checked_out_to.lower().replace('.', '_')
        if clean_username not in overall_workload:
            overall_workload[clean_username] = {}

        if 'pipes' not in overall_workload[clean_username]:
            overall_workload[clean_username]['pipes']: int = 0

        if 'pipes' in overall_workload[clean_username]:
            overall_workload[clean_username]['pipes'] += 1

        # Create pipe data structure within all the Console Server data
        console_server_data[host_group_name]: dict = {}

        # Host Groups that are actually getting data from
        # if 'Pipe-' in host_group_name and 'OFFLINE' not in host_group_name and 'OFFLINE' not in status:

        # Tells users progress
        print(f'\t\t- Collect  |  {Fore.GREEN}Success{Style.RESET_ALL}   |  {host_group_name}')
        log_total.append(1)
        log_collect.append(1)

        # Get all system data from Pipe
        current_pipe: dict = console_server_data[host_group_name]
        console_server_data[host_group_name]['pipe_data']: dict = {}

        # Stores
        current_pipe['pipe_data']: dict = {}

        current_pipe['pipe_data']['pipe_inventory']: dict = {}
        current_pipe['pipe_data']['pipe_inventory']['dimms']: dict = {}
        current_pipe['pipe_data']['pipe_inventory']['nvmes']: dict = {}
        current_pipe['pipe_data']['pipe_inventory']['disks']: dict = {}

        # Stores total systems in Pipe (non-vm or anything else)
        current_pipe['setup_data']: dict = {}
        current_pipe['setup_data']['total_systems']: dict = []
        current_pipe['setup_data']['systems_with_ticket']: dict = []

        # Runs async and stores data in module-level dictionary
        try:
            asyncio.run(start_loop(host_ids, host_group_name))
        except aiohttp.client_exceptions.ClientPayloadError:
            return main_method()

        # Store all system data from Pipe
        current_pipe['description']: str = description
        current_pipe['host_group_status']: str = status
        current_pipe['host_id']: str = host_id
        current_pipe['host_ids']: list = host_ids
        current_pipe['checked_out_to']: str = checked_out_to

        # Get and store ticket information
        group_unique_tickets: list = list(set(current_pipe_tickets))
        current_pipe['group_unique_tickets']: list = group_unique_tickets
        current_pipe_tickets.clear()

        # Store information for setup dashboard later
        current_pipe['ticket_tally']: dict = {}
        current_pipe['ticket_tally']['systems_with_tickets']: dict = sum(total_tickets_in_pipe)
        current_pipe['ticket_tally']['total_systems']: dict = sum(total_systems_in_pipe)
        current_pipe['ticket_tally']['total_vms']: dict = sum(total_vms)

        # Clear information on pipe tally for next iteration
        total_vms.clear()
        total_tickets_in_pipe.clear()
        total_systems_in_pipe.clear()

    # Stores host groups level information for Setup information later
    console_server_data['host_groups_data']: dict = {}
    console_server_data['host_groups_data']['all_unique_tickets']: dict = list(set(all_tickets))
    console_server_data['host_groups_data']['log_total']: str = str(sum(log_total))
    console_server_data['host_groups_data']['log_skipped']: str = str(sum(log_skipped))
    console_server_data['host_groups_data']['log_collect']: str = str(sum(log_collect))
    console_server_data['host_groups_data']['log_non_pipes']: str = str(sum(log_non_pipes))
    console_server_data['host_groups_data']['log_off_lines']: str = str(sum(log_off_lines))
    console_server_data['host_groups_data']['log_idle']: str = str(sum(log_idle))

    console_server_data['dhcp_data']: list = get_dhcp()

    # Sorted to keep consistent alphabetic order
    console_server_data['virtual_machine_data']: list = virtual_machine_data

    console_server_data['all_serial']: list = all_serial
    console_server_data['all_part_numbers']: list = all_part_numbers

    return console_server_data


""" Helper functions for cci_plot.py.
"""

import sys
import math
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.axes import Axes
from matplotlib.patches import Arc, Wedge
from mpl_toolkits.axes_grid1 import make_axes_locatable

from matplotlib.path import Path
import matplotlib.patches as patches
import matplotlib.colors as plt_colors
import matplotlib.cm as cm

from ..tools.microenv.cci.het import get_edges

from anndata import AnnData

# Helper functions for overview plots of the LRs.

def lr_scatter(
    data,
    feature,
    highlight_lrs=None,
    show_text=True,
    n_top=50,
    color="gold",
    alpha=0.5,
    lr_text_fp=None,
    axis_text_fp=None,
    ax=None,
    show=True,
    max_text=100,
    highlight_color="red",
    figsize: tuple = None,
    show_all: bool = False,
):
    """General plotting of the LR features."""
    highlight = type(highlight_lrs) != type(None)
    if not highlight:
        show_text = show_text if n_top <= max_text else False
    else:
        highlight_lrs = highlight_lrs[0:max_text]

    lr_df = data.uns["lr_summary"]
    lrs = lr_df.index.values.astype(str)[0:n_top]
    lr_features = data.uns["lrfeatures"]
    lr_df = pd.concat([lr_df, lr_features], axis=1).loc[lrs, :]
    if feature not in lr_df.columns:
        raise Exception(f"Inputted {feature}; must be one of " f"{list(lr_df.columns)}")

    rot = 90 if feature != "n_spots_sig" else 70

    n_spots = lr_df.loc[:, feature].values[0:n_top]

    return rank_scatter(
        lrs,
        n_spots,
        y_label=feature,
        x_label="LR Rank (n_sig_spots)",
        figsize=figsize,
        highlight_items=highlight_lrs,
        show_text=show_text,
        color=color,
        alpha=alpha,
        lr_text_fp=lr_text_fp,
        axis_text_fp=axis_text_fp,
        ax=ax,
        show=show,
        highlight_color=highlight_color,
        rot=rot,
        pad=0,
        show_all=show_all,
    )
    # ranks = np.array(list(range(len(n_spots))))
    #
    # if type(lr_text_fp)==type(None):
    #     lr_text_fp = {'weight': 'bold', 'size': 8}
    # if type(axis_text_fp)==type(None):
    #     axis_text_fp = {'weight': 'bold', 'size': 12}
    #
    # if type(ax)==type(None):
    #     width = (7.5 / 50) * n_top if show_text and not highlight else 7.5
    #     if width > 20:
    #         width = 20
    #     fig, ax = plt.subplots(figsize=(width, 4))
    #
    # # Plotting the points #
    # ax.scatter(ranks, n_spots, alpha=alpha, c=color)
    #
    # if show_text:
    #     if highlight:
    #         ranks = ranks[[np.where(lrs==lr)[0][0] for lr in highlight_lrs]]
    #         ax.scatter(ranks, n_spots[ranks], alpha=alpha, c=highlight_color)
    #
    #     for i in ranks:
    #         ax.text(i-.2, n_spots[i], lrs[i], rotation=rot, fontdict=lr_text_fp)
    #
    # ax.spines['top'].set_visible(False)
    # ax.spines['right'].set_visible(False)
    # ax.set_xlabel('LR Rank', axis_text_fp)
    # ax.set_ylabel(feature, axis_text_fp)
    #
    # if show:
    #     plt.show()
    # else:
    #     return ax

def rank_scatter(
    items,
    y,
    y_label: str = "",
    x_label: str = "",
    highlight_items=None,
    show_text=True,
    color="gold",
    alpha=0.5,
    lr_text_fp=None,
    axis_text_fp=None,
    ax=None,
    show=True,
    highlight_color="red",
    rot: float = 90,
    point_sizes: np.array = None,
    pad=0.2,
    figsize=None,
    width_ratio=7.5 / 50,
    height=4,
    point_size_name="Sizes",
    point_size_exp=2,
    show_all: bool = False,
):
    """General plotting function for showing ranked list of items."""
    ranks = np.array(list(range(len(items))))

    highlight = type(highlight_items) != type(None)
    if type(lr_text_fp) == type(None):
        lr_text_fp = {"weight": "bold", "size": 8}
    if type(axis_text_fp) == type(None):
        axis_text_fp = {"weight": "bold", "size": 12}

    if type(ax) == type(None):
        if type(figsize) == type(None):
            width = width_ratio * len(ranks) if show_text and not highlight else 7.5
            if width > 20:
                width = 20
            figsize = (width, height)
        fig, ax = plt.subplots(figsize=figsize)

    # Plotting the points #
    scatter = ax.scatter(
        ranks,
        y,
        alpha=alpha,
        c=color,
        s=None if type(point_sizes) == type(None) else point_sizes ** point_size_exp,
        edgecolors="none",
    )
    y_min, y_max = ax.get_ylim()
    y_max = y_max + y_max * pad
    ax.set_ylim(y_min, y_max)
    if type(point_sizes) != type(None):
        # produce a legend with a cross section of sizes from the scatter
        handles, labels = scatter.legend_elements(prop="sizes", alpha=0.6, num=4)
        [handle.set_markeredgecolor("none") for handle in handles]
        starts = [label.find("{") for label in labels]
        ends = [label.find("}") + 1 for label in labels]
        sizes = [
            float(label[(starts[i] + 1) : (ends[i] - 1)])
            for i, label in enumerate(labels)
        ]
        counts = [int(size ** (1 / point_size_exp)) for size in sizes]
        labels2 = [
            label.replace(label[(starts[i]) : (ends[i])], "{" + str(counts[i]) + "}")
            for i, label in enumerate(labels)
        ]
        legend2 = ax.legend(
            handles,
            labels2,
            frameon=False,
            # bbox_to_anchor=(0.1, 0.05, 1., 1.),
            handletextpad=1.6,
            loc="upper right",
            title=point_size_name,
        )

    if show_text:
        if highlight:
            ranks_ = ranks[[np.where(items == item)[0][0] for item in highlight_items]]
            ax.scatter(
                ranks_,
                y[ranks_],
                alpha=alpha,
                c=highlight_color,
                s=None
                if type(point_sizes) == type(None)
                else (point_sizes[ranks_] ** point_size_exp),
                edgecolors=color,
            )
            ranks = ranks_ if not show_all else ranks

        for i in ranks:
            ax.text(i - 0.2, y[i], items[i], rotation=rot, fontdict=lr_text_fp)

    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.set_xlabel(x_label, axis_text_fp)
    ax.set_ylabel(y_label, axis_text_fp)

    if show:
        plt.show()
    else:
        return ax

# Helper functions for lr_plot

def add_arrows(
    adata: AnnData,
    l_expr: np.array,
    r_expr: np.array,
    min_expr: float,
    sig_bool: np.array,
    fig,
    ax: Axes,
    use_label: str,
    int_df: pd.DataFrame,
    head_width=4,
    width=0.001,
    arrow_cmap=None,
    arrow_vmax=None,
):
    """ Adds arrows to the current plot for significant spots to neighbours \
        which is interacting with.
        Parameters
        ----------
        adata: AnnData          The anndata object.
        L_bool: np.array
        Returns
        -------
        counts: int   Total number of interactions satisfying the conditions, \
                      or np.array<set> if return_edges=True, where each set is \
                      an edge, only returns unique edges.
    """

    library_id = list(adata.uns["spatial"].keys())[0]
    res = adata.uns["spatial"][library_id]["use_quality"]
    scale_factor = adata.uns["spatial"][library_id]["scalefactors"][
        f"tissue_{res}_scalef"
    ]
    # scale_factor = 1
    # scale_factor = 1 # scale factor pretty much isn't used since always parsed as 1
    # in the base plotting function class.
    # Reason why is because scale_factor refers to scaling the
    # image to match the spot spatial coordinates, not the
    # the spots to match the image coordinates!!!

    L_bool = l_expr > min_expr
    R_bool = r_expr > min_expr

    # Getting the edges, from sig-L->R and sig-R<-L #
    forward_edges, reverse_edges = get_edges(adata, L_bool, R_bool, sig_bool)

    # If int_df specified, means need to subset to CCIs which are significant #
    if type(int_df) != type(None):
        spot_bcs = adata.obs_names.values.astype(str)
        spot_labels = adata.obs[use_label].values.astype(str)
        label_set = int_df.index.values.astype(str)
        interact_bool = int_df.values > 0

        # Subsetting to only significant CCI #
        edges_sub = [[], []]  # forward, reverse
        # ints_2 = np.zeros(int_df.shape) # Just for debugging make sure edge
        # list re-capitulates edge-counts.
        for i, edges in enumerate([forward_edges, reverse_edges]):
            for j, edge in enumerate(edges):
                k_ = [0, 1] if i == 0 else [1, 0]
                celltype0 = np.where(label_set == spot_labels[spot_bcs == edge[k_[0]]])[
                    0
                ][0]
                celltype1 = np.where(label_set == spot_labels[spot_bcs == edge[k_[1]]])[
                    0
                ][0]
                celltypes = np.array([celltype0, celltype1])
                # For debugging purposes, used to find indexing bug #
                # print(label_set[celltypes[k_[0]]],
                #       label_set[celltypes[k_[1]]])
                # print( celltypes[k_[0]], celltypes[k_[1]] )
                # print( interact_bool[celltypes[k_[0]], celltypes[k_[1]]] )
                if interact_bool[celltypes[k_[0]], celltypes[k_[1]]]:
                    edges_sub[i].append(edge)
                    # ints_2[celltypes[k_[0]], celltypes[k_[1]]] += 1

        forward_edges, reverse_edges = edges_sub

    # If cmap specified, colour arrows by average LR expression on edge #
    if type(arrow_cmap) != type(None):
        edges_means = [[], []]
        all_means = []
        for i, edges in enumerate([forward_edges, reverse_edges]):
            for j, edge in enumerate(edges):
                edge0_bool = spot_bcs == edge[0]
                edge1_bool = spot_bcs == edge[1]
                l_expr0 = l_expr[edge0_bool]
                l_expr1 = l_expr[edge1_bool]
                r_expr0 = r_expr[edge0_bool]
                r_expr1 = r_expr[edge1_bool]
                mean_expr = np.mean([l_expr0, l_expr1, r_expr0, r_expr1])
                edges_means[i].append(mean_expr)
                all_means.append(mean_expr)

        # Determining the color maps #
        arrow_vmax = np.max(all_means) if type(arrow_vmax) == type(None) else arrow_vmax
        cmap = plt.get_cmap(arrow_cmap)
        c_norm = plt_colors.Normalize(vmin=0, vmax=arrow_vmax)
        scalar_map = cm.ScalarMappable(norm=c_norm, cmap=cmap)

        # Determining the edge colors #
        edges_colors = [[], []]
        for i, edges in enumerate([forward_edges, reverse_edges]):
            for j, edge in enumerate(edges):
                color_val = scalar_map.to_rgba(edges_means[i][j])
                edges_colors[i].append(color_val)

        # Need to add new axes #
        # xlims = ax.get_xlim()
        # ylims = ax.get_ylim()
        # left, bottom = xlims[0]*0.025, ylims[0]*0.17
        # axc = fig.add_axes([0, 0, 0.28, 0.015])
        divider = make_axes_locatable(ax)
        cax = divider.new_vertical(size="2%", pad=0.1, pack_start=True)
        axc = fig.add_axes(cax)

    else:
        edges_colors = [None, None]

    # Now performing the plotting #
    # The arrows #
    # Now converting the edges to coordinates #
    add_arrows_by_edges(
        ax,
        adata,
        forward_edges,
        scale_factor,
        head_width,
        width,
        edge_colors=edges_colors[0],
    )
    add_arrows_by_edges(
        ax,
        adata,
        reverse_edges,
        scale_factor,
        head_width,
        width,
        forward=False,
        edge_colors=edges_colors[1],
    )
    # Adding the color map #
    if type(arrow_cmap) != type(None):
        cb1 = matplotlib.colorbar.ColorbarBase(
            axc, cmap=cmap, norm=c_norm, orientation="horizontal"
        )

def add_arrows_by_edges(
    ax,
    adata,
    edges,
    scale_factor,
    head_width,
    width,
    forward=True,
    edge_colors=None,
    axc=None,
):
    """Adds the arrows using an edge list."""
    for i, edge in enumerate(edges):
        # cols = ["imagecol", "imagerow"]
        if forward:
            edge0, edge1 = edge
        else:
            edge0, edge1 = edge[::-1]

        # Arrow details #
        # x1, y1 = adata.obs.loc[edge0, cols].values.astype(float) * scale_factor
        # x2, y2 = adata.obs.loc[edge1, cols].values.astype(float) * scale_factor
        edge0_index = np.where(adata.obs_names.values == edge0)[0][0]
        edge1_index = np.where(adata.obs_names.values == edge1)[0][0]
        x1, y1 = adata.obsm["spatial"][edge0_index, :] * scale_factor
        x2, y2 = adata.obsm["spatial"][edge1_index, :] * scale_factor
        dx, dy = (x2 - x1) * 0.75, (y2 - y1) * 0.75
        arrow_color = "k" if type(edge_colors) == type(None) else edge_colors[i]

        ax.arrow(
            x1,
            y1,
            dx,
            dy,
            head_width=head_width,
            width=width,
            linewidth=0.01,
            facecolor=arrow_color,
        )

# Helper functions for cci_map

def get_int_df(adata, lr, use_label, sig_interactions, title):
    """Retrieves the relevant interaction count matrix."""
    no_title = type(title) == type(None)
    labels_ordered = adata.obs[use_label].cat.categories
    if type(lr) == type(None):  # No LR inputted, so just use all
        int_df = (
            adata.uns[f"lr_cci_{use_label}"]
            if sig_interactions
            else adata.uns[f"lr_cci_raw_{use_label}"]
        )[labels_ordered].loc[labels_ordered]
        title = "Cell-Cell LR Interactions" if no_title else title
    else:

        labels_ordered = adata.obs[use_label].cat.categories
        int_df = (
            adata.uns[f"per_lr_cci_{use_label}"][lr]
            if sig_interactions
            else adata.uns[f"per_lr_cci_raw_{use_label}"][lr]
        )[labels_ordered].loc[labels_ordered]

        title = f"Cell-Cell {lr} interactions" if no_title else title

    return int_df, title

def create_flat_df(int_df):
    """Reformats a dataframe representing interactions to a flat format."""
    n_rows = int_df.shape[0] * int_df.shape[1]
    flat_df = pd.DataFrame(index=list(range(n_rows)), columns=["x", "y", "value"])
    row_i = 0
    for i, cell_typei in enumerate(int_df.index.values):
        for j, cell_typej in enumerate(int_df.columns.values):
            flat_df.iloc[row_i, :] = [cell_typei, cell_typej, int_df.values[i, j]]
            row_i += 1

    return flat_df

def _box_map(x, y, size, ax=None, figsize=(6.48, 4.8), cmap=None, square_scaler=700):
    """Main underlying helper function for generating the heatmaps."""
    if type(cmap) == type(None):
        cmap = "Spectral_r"

    if type(ax) == type(None):
        fig, ax = plt.subplots(figsize=figsize)

    # Mapping from column names to integer coordinates
    x_labels = list(x.values)  # [v for v in sorted(x.unique())]
    y_labels = list(y.values)  # [v for v in sorted(y.unique())]
    x_to_num = {p[1]: p[0] for p in enumerate(x_labels)}
    y_to_num = {p[1]: p[0] for p in enumerate(y_labels)}

    out = ax.scatter(
        x=x.map(x_to_num),  # Use mapping for x
        y=y.map(y_to_num),  # Use mapping for y
        s=size / sum(size) * square_scaler,
        c=size,
        cmap=cmap,
        # Vector of square sizes, proportional to size parameter
        marker="s",  # Use square as scatterplot marker
    )
    out.set_array(size.values.astype(int))
    out.set_clim(min(size), max(size))
    cbar = plt.colorbar(out)
    cbar.ax.get_yaxis().labelpad = 15
    cbar.ax.set_ylabel("# of interactions", rotation=270)

    # Show column labels on the axes
    ax.set_xticks([x_to_num[v] for v in x_labels])
    ax.set_xticklabels(x_labels, rotation=45, horizontalalignment="right")
    ax.set_yticks([y_to_num[v] for v in y_labels])
    ax.set_yticklabels(y_labels)

    return ax

# Matplotlib chord diagram, adapted & refactored from:
# https://github.com/fengwangPhysics/matplotlib-chord-diagram/blob/master/matplotlib-chord.py

###################
# chord diagram
LW = 0.3

def polar2xy(r, theta):
    return np.array([r * np.cos(theta), r * np.sin(theta)])

def hex2rgb(c):
    return tuple(int(c[i : i + 2], 16) / 256.0 for i in (1, 3, 5))

def IdeogramArc(
    start=0, end=60, radius=1.0, width=0.2, ax=None, color=(1, 0, 0), curve_steps=1
):
    # start, end should be in [0, 360)
    if start > end:
        start, end = end, start
    start *= np.pi / 180.0
    end *= np.pi / 180.0
    # optimal distance to the control points
    # https://stackoverflow.com/questions/1734745/how-to-create-circle-with-b%C3%A9zier-curves
    opt = 4.0 / 3.0 * np.tan((end - start) / 4.0) * radius
    inner = radius * (1 - width)

    # Defines descrete points along the arc #
    # The outter part of the arc #

    val = 0.5
    verts_upper_start = [polar2xy(radius, start)]
    verts_upper_curve = [
        polar2xy(radius, start)
        + polar2xy(opt, start + (np.pi * ((val * i) / curve_steps)))
        for i in range(1, curve_steps + 1)
    ]
    verts_upper_curve += [
        polar2xy(radius, end) + polar2xy(opt, end - (np.pi * ((val * i) / curve_steps)))
        for i in range(1, curve_steps + 1)
    ]
    verts_upper = verts_upper_start + verts_upper_curve + [polar2xy(radius, end)]

    verts_inner_start = [polar2xy(inner, end)]
    verts_inner_curve = [
        polar2xy(inner, end)
        + polar2xy(opt * (1 - width), end - (np.pi * ((val * i) / curve_steps)))
        for i in range(1, curve_steps + 1)
    ]
    verts_inner_curve += [
        polar2xy(inner, start)
        + polar2xy(opt * (1 - width), start + (np.pi * ((val * i) / curve_steps)))
        for i in range(1, curve_steps + 1)
    ]
    verts_inner = (
        verts_inner_start
        + verts_inner_curve
        + [polar2xy(inner, start), polar2xy(radius, start)]
    )

    verts = verts_upper + verts_inner

    codes = (
        [Path.MOVETO]
        + [Path.CURVE4] * curve_steps * 2
        + [Path.CURVE4, Path.LINETO]
        + [Path.CURVE4] * curve_steps * 2
        + [
            Path.CURVE4,
            Path.CLOSEPOLY,
        ]
    )

    if ax == None:
        return verts, codes
    else:
        path = Path(verts, codes)
        patch = patches.PathPatch(
            path, facecolor=color + (0.5,), edgecolor=color + (0.4,), lw=LW
        )
        ax.add_patch(patch)

def ChordArc(
    start1=0,
    end1=60,
    start2=180,
    end2=240,
    radius=1.0,
    chordwidth=0.7,
    ax=None,
    color=(1, 0, 0),
):
    # start, end should be in [0, 360)
    if start1 > end1:
        start1, end1 = end1, start1
    if start2 > end2:
        start2, end2 = end2, start2
    start1 *= np.pi / 180.0
    end1 *= np.pi / 180.0
    start2 *= np.pi / 180.0
    end2 *= np.pi / 180.0
    opt1 = 4.0 / 3.0 * np.tan((end1 - start1) / 4.0) * radius
    opt2 = 4.0 / 3.0 * np.tan((end2 - start2) / 4.0) * radius
    rchord = radius * (1 - chordwidth)
    verts = [
        polar2xy(radius, start1),
        polar2xy(radius, start1) + polar2xy(opt1, start1 + 0.5 * np.pi),
        polar2xy(radius, end1) + polar2xy(opt1, end1 - 0.5 * np.pi),
        polar2xy(radius, end1),
        polar2xy(rchord, end1),
        polar2xy(rchord, start2),
        polar2xy(radius, start2),
        polar2xy(radius, start2) + polar2xy(opt2, start2 + 0.5 * np.pi),
        polar2xy(radius, end2) + polar2xy(opt2, end2 - 0.5 * np.pi),
        polar2xy(radius, end2),
        polar2xy(rchord, end2),
        polar2xy(rchord, start1),
        polar2xy(radius, start1),
    ]

    codes = [
        Path.MOVETO,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
    ]

    if ax == None:
        return verts, codes
    else:
        path = Path(verts, codes)
        patch = patches.PathPatch(
            path, facecolor=color + (0.5,), edgecolor=color + (0.4,), lw=LW
        )
        ax.add_patch(patch)

def selfChordArc(start=0, end=60, radius=1.0, chordwidth=0.7, ax=None, color=(1, 0, 0)):
    # start, end should be in [0, 360)
    if start > end:
        start, end = end, start
    start *= np.pi / 180.0
    end *= np.pi / 180.0
    opt = 4.0 / 3.0 * np.tan((end - start) / 4.0) * radius
    rchord = radius * (1 - chordwidth)
    verts = [
        polar2xy(radius, start),
        polar2xy(radius, start) + polar2xy(opt, start + 0.5 * np.pi),
        polar2xy(radius, end) + polar2xy(opt, end - 0.5 * np.pi),
        polar2xy(radius, end),
        polar2xy(rchord, end),
        polar2xy(rchord, start),
        polar2xy(radius, start),
    ]

    codes = [
        Path.MOVETO,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
        Path.CURVE4,
    ]

    if ax == None:
        return verts, codes
    else:
        path = Path(verts, codes)
        patch = patches.PathPatch(
            path, facecolor=color + (0.5,), edgecolor=color + (0.4,), lw=LW
        )
        ax.add_patch(patch)

def chordDiagram(X, ax, colors=None, width=0.1, pad=2, chordwidth=0.7, lim=1.1):
    """Plot a chord diagram
    Parameters
    ----------
    X :
        flux data, X[i, j] is the flux from i to j
    ax :
        matplotlib `axes` to show the plot
    colors : optional
        user defined colors in rgb format. Use function hex2rgb() to convert hex color to rgb color. Default: d3.js category10
    width : optional
        width/thickness of the ideogram arc
    pad : optional
        gap pad between two neighboring ideogram arcs, unit: degree, default: 2 degree
    chordwidth : optional
        position of the control points for the chords, controlling the shape of the chords
    """
    # X[i, j]:  i -> j
    x = X.sum(axis=1)  # sum over rows
    ax.set_xlim(-lim, lim)
    ax.set_ylim(-lim, lim)
    diam = 1.8

    if colors is None:
        # use d3.js category10 https://github.com/d3/d3-3.x-api-reference/blob/master/Ordinal-Scales.md#category10
        colors = [
            "#1f77b4",
            "#ff7f0e",
            "#2ca02c",
            "#d62728",
            "#9467bd",
            "#8c564b",
            "#e377c2",
            "#7f7f7f",
            "#bcbd22",
            "#17becf",
        ]
        if len(x) > 10:
            print("x is too large! Use x smaller than 10")
    if type(colors[0]) == str:
        colors = [hex2rgb(colors[i]) for i in range(len(x))]

    # find position for each start and end
    y = x / np.sum(x).astype(float) * (360 - pad * len(x))

    pos = {}
    arc = []
    nodePos = []
    start = 0
    for i in range(len(x)):
        end = start + y[i]
        arc.append((start, end))
        angle = 0.5 * (start + end)
        # print(start, end, angle)
        if -30 <= angle <= 210:
            angle -= 90
        else:
            angle -= 270
        nodePos.append(
            tuple(
                polar2xy((diam / 2) + diam * 0.05, 0.5 * (start + end) * np.pi / 180.0)
            )
            + (angle,)
        )
        z = (X[i, :] / x[i].astype(float)) * (end - start)
        ids = np.argsort(z)
        z0 = start
        for j in ids:
            pos[(i, j)] = (z0, z0 + z[j])
            z0 += z[j]
        start = end + pad

    for i in range(len(x)):
        start, end = arc[i]
        # This draws the outter ring #
        # IdeogramArc(start=start, end=end, radius=1.0, ax=ax,
        #            color=colors[i], width=width)
        a = Arc((0, 0), diam, diam, 0, start, end, color=colors[i], lw=10)
        ax.add_patch(a)
        start, end = pos[(i, i)]
        # This draws the paths to itself #
        if end - start < 180:  # Indicates this method will work fine !
            selfChordArc(
                start,
                end,
                radius=1.0 - width,
                color=colors[i],
                chordwidth=chordwidth * 0.7,
                ax=ax,
            )
        else:  # Need to use a wedge because the arch distorts past 180-degrees
            path = Wedge(0, diam / 2, start, end, color=colors[i] + (0.5,))
            ax.add_patch(path)
        for j in range(i):
            if X[i, j] == 0 and X[j, i] == 0:  # don't draw anything for no interaction
                continue
            color = colors[i]
            if X[i, j] > X[j, i]:  # Color by the dominant signal #
                color = colors[j]
            start1, end1 = pos[(i, j)]
            start2, end2 = pos[(j, i)]
            ChordArc(
                start1,
                end1,
                start2,
                end2,
                radius=1.0 - width,
                color=color,
                chordwidth=chordwidth,
                ax=ax,
            )

    return nodePos


#Implementation of Linear regression with machine learning scikit learn
import numpy
import pandas
import matplotlib.pyplot as plt
import os
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# print(os.getcwd())
# os.chdir('/Users/harshshetye/Desktop')
# print(os.getcwd())

# names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class']
# data = pandas.read_csv('iris (1).csv', names=names)

# x = data['sepal-length'].values
# y = data['sepal-width'].values
# #cannot use rank 1 matrix in scikit learn
# n = len(x)
# x = x.reshape((n,1)) #making a matrix

# #Creating model
# reg = LinearRegression()
# #Fitting trainig data
# reg = reg.fit(x, y)

# #Y prediction
# y_pre = reg.predict(x)

# #Calculating R2 score
# R2 = reg.score(x, y)

# print(R2)

rng = numpy.random.RandomState(1)
x = 10 * rng.rand(50)
y = 2*x - 5 + rng.rand(50)

plt.scatter(x, y)
plt.show()

model = LinearRegression(fit_intercept=True)
model.fit(x[:, numpy.newaxis], y)
xfit = numpy.linspace(0, 10, 1000)
yfit = model.predict(xfit[:, numpy.newaxis])

plt.scatter(x, y)
plt.plot(xfit, yfit)

plt.show()

print('Model slope : ', model.coef_)
print('Model Intercept : ', model.intercept_)

#Testing with multidimensional data

rng = numpy.random.RandomState(1)
X = 10 * rng.rand(100, 3)
y = 0.5 + numpy.dot(X, [1.5, -2., 1.])

model.fit(X, y)
print('Model slope : ', model.coef_)
print('Model intercept : ', model.intercept_)


# -*- coding: utf-8 -*-
"""
Задание 21.2

Сделать шаблон TextFSM для обработки вывода sh ip dhcp snooping binding
и записать его в файл templates/sh_ip_dhcp_snooping.template

Вывод команды находится в файле output/sh_ip_dhcp_snooping.txt.

Шаблон должен обрабатывать и возвращать значения таких столбцов:
* mac - такого вида 00:04:A3:3E:5B:69
* ip - такого вида 10.1.10.6
* vlan - 10
* intf - FastEthernet0/10

Проверить работу шаблона с помощью функции parse_command_output из задания 21.1.
"""
from task_22_1 import parse_command_output

if __name__ == "__main__":
    with open("output/sh_ip_dhcp_snooping.txt") as show:
        output = show.read()
    result = parse_command_output("templates/sh_ip_dhcp_snooping.template", output)
    print(result)

# templates/sh_ip_dhcp_snooping.template
"""
Value mac (\S+)
Value ip (\S+)
Value vlan (\d+)
Value intf (\S+)

Start
  ^${mac}\s+${ip}\s+\d+\s+\S+\s+${vlan}\s+${intf} -> Record
"""


#! C:\Users\yeshw\Desktop\Umass\Geology_497\Lab4
"""
This code extracts GPS data from two locations in iceland and creates three
subplots showing the north, east and vertical positions of the GPS data.

Yesh 2/26/2020
"""
import numpy as np
import matplotlib.pylab as plt

fname1 = 'HOFN.IGS08.tenv3'
fname2 = 'RYEK.IGS08.tenv3'

dtype1 = np.dtype([('East position', 'f8')
, ('North position', 'f8'), ('Vertical position', 'f8')])

data_HOFN = np.loadtxt(fname1, dtype=dtype1, skiprows=1, usecols=(8,10,12))
data_REYK = np.loadtxt(fname2, dtype=dtype1, skiprows=1, usecols=(8,10,12))

def getGPSstats(data):
    numberOfDays = len(data['East position'])
    totalTime = numberOfDays/365
    totalEastDisplacement = (data['East position'][-1] - data['East position'][0])*100
    totalNorthDisplacement = (data['North position'][-1] - data['North position'][0])*100
    totalVerticalDisplacement = (data['Vertical position'][-1] - data['Vertical position'][0])*100
    averageEastVelocity = totalEastDisplacement/totalTime
    averageNorthVelocity = totalNorthDisplacement/totalTime
    averageVerticalVelocity = totalVerticalDisplacement/totalTime

    print('Time Span: ','{:.4}'.format(totalTime),'years')
    print('Number of days with data: ',numberOfDays,'years')
    print('Total north displacement: ','{:.4}'.format(totalNorthDisplacement),'cm')
    print('Total east displacement: ','{:.4}'.format(totalEastDisplacement),'cm')
    print('Total vertica displacement: ','{:.4}'.format(totalVerticalDisplacement),'cm')
    print('Average north displacement: ','{:.2}'.format(averageNorthVelocity),'cm/year')
    print('Average east displacement: ','{:.2}'.format(averageEastVelocity),'cm/year')
    print('Average vertical displacement: ','{:.2}'.format(averageVerticalVelocity),'cm/year')
    print('')

getGPSstats(data_HOFN)
getGPSstats(data_REYK)

dates1 = np.arange(0,len(data_HOFN['East position']))
dates2 = np.arange(0,len(data_REYK['East position']))

fig = plt.figure(figsize=(10,10))

plt.figure(1)
plt.subplot(3, 1, 1)
plt.plot(dates1,data_HOFN['North position'], color = 'red', label = 'HOFN')
plt.plot(dates2,data_REYK['North position'], color = 'blue', label = 'REYK')
plt.grid(True)
plt.xlabel('Number of days since recording data')
plt.ylabel('North Position')
plt.legend(loc="upper left")

plt.subplot(3, 1, 2)
plt.plot(dates1,data_HOFN['East position'], color = 'red', label = 'HOFN')
plt.plot(dates2,data_REYK['East position'], color = 'blue', label = 'REYK')
plt.grid(True)
plt.xlabel('Number of days since recording data')
plt.ylabel('East Position')
plt.legend(loc="upper left")

plt.subplot(3, 1, 3)
plt.plot(dates1,data_HOFN['Vertical position'], color = 'red', label = 'HOFN')
plt.plot(dates2,data_REYK['Vertical position'], color = 'blue', label = 'REYK')
plt.grid(True)
plt.xlabel('Number of days since recording data')
plt.ylabel('Vertical Position')
plt.legend(loc="upper left")

plt.tight_layout()
plt.show()


# prime number
# count=0
# l=[]
# for n in range(2,100):
#     if count<20:
#         for i in range(2,round(n/2)+1):
#             if(n%i == 0):
#               break;
#
#         else:
#            l.append(n)
#            count+=1
#
# print(l)

# count number of digits in number
# num=int(input("enter number"))
# count=0
# while(num>0):
#     num=num//10;
#     count+=1
#
#
# print(count)

# reverse number
# rnumber=0
# num=1234
# while(num>0):
#     n=num%10
#     rnumber=rnumber*10 + n
#     num=num//10
#
# print(rnumber)
#
# #prime no not
#
# num=41
# flag=True
# for i in range(2,round(num/2)+1):
#     if(num%i==0):
#         flag=False
#
# if (flag==True):
#     print(f"{num} is prime number")
# else:
#     print(f"{num} is not prime number")
#
# # reverse string for loop
# string="sham"
# rstr=""
# for i in string:
#     rstr=i + rstr
#
# print(rstr)

# using slice operator
p="aman"

print(p[::-1])
# revrsed string method
print("".join(reversed(p)))

# convert list to string
list=['h','e','l','l','o']

print("".join(list))
type("".join(list))

print(tuple(list))
type(tuple(list))

print(set(list))
type(set(list))

# fibonacci seris

a=0
b=1
count=0
l=[]
while(count<15):
    l.append(a)
    c=a+b
    a=b
    b=c
    count+=1

print(l )


#Matthew Leung
#February 2022

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import cv2 as cv
import time
from Gaussian_Fit import find_centroid_iter, crop_image_for_fit, fit_gaussian, gaussian, gaussian_FWHM
from pixelink_class import pixelink_class as pixelink_class

if __name__ == "__main__":
    camera_obj = pixelink_class()
    camera_obj.init_camera()
    camera_obj.set_exposure(exp_time=0.01)

    def take_frame():
        #take each frame
        return camera_obj.get_snapshot_np_array('jpg')

    img = take_frame()

    start_time = time.time() #starting time

    x_cen, y_cen = find_centroid_iter(img, cen_tol=5, maxiter=10, do_plot=False)
    cropped_img, crop_radius = crop_image_for_fit(img, x_cen, y_cen, radius_scale=3)
    h, w = cropped_img.shape

    #np.savetxt("error.txt", cropped_img)
    #plt.figure()
    #plt.imshow(cropped_img)
    #plt.show()

    #Fit Gaussian
    params, params_uncert, params_dict, params_uncert_dict, fit_fcn = fit_gaussian(cropped_img)
    FWHM_x = gaussian_FWHM(params_dict['sigma_x'])
    FWHM_y = gaussian_FWHM(params_dict['sigma_y'])
    FWHM_avg = (FWHM_x + FWHM_y)/2
    x = np.arange(cropped_img.shape[1])
    y = np.arange(cropped_img.shape[0])
    X, Y = np.meshgrid(x, y)
    Z = gaussian(X, Y, *params)

    fig, ax = plt.subplots(nrows=1, ncols=4, figsize=(16,4), constrained_layout=True)

    #Plot the original image
    im0 = ax[0].imshow(img, cmap='gray')
    rect = patches.Rectangle((x_cen-crop_radius, y_cen-crop_radius), crop_radius*2, crop_radius*2, linewidth=1, edgecolor='r', facecolor='none')
    ax[0].add_patch(rect)
    ax[0].set_title('Original Image')

    #Plot the cropped region
    im1 = ax[1].imshow(cropped_img, cmap='gray')
    ax[1].set_title('Cropped Region')
    ax[1].get_xaxis().set_visible(False) #Hide ticks
    ax[1].get_yaxis().set_visible(False)

    #Plot the fitted Gaussian
    im2 = ax[2].imshow(Z, cmap='gray')
    ax[2].set_title('Gaussian Fit')
    ax[2].get_xaxis().set_visible(False) #Hide ticks
    ax[2].get_yaxis().set_visible(False)
    plot_text_x = "FWHM$_{x}$ = " + "{:.3f} pixels".format(FWHM_x)
    plot_text_y = "FWHM$_{y}$ = " + "{:.3f} pixels".format(FWHM_y)
    tx = ax[2].text(x=0.02,y=0.07, s=plot_text_x, fontsize=24, transform=ax[2].transAxes, color='r')
    ty = ax[2].text(x=0.02,y=-0.05, s=plot_text_y, fontsize=24, transform=ax[2].transAxes, color='limegreen')

    #Plot time VS average FWHM
    prepoints = 50 #number of points before start of data acquisition
    time_array = np.linspace(-10,0,num=prepoints)
    FWHM_array = np.zeros(prepoints)
    time_plot, = ax[3].plot(time_array,FWHM_array, 'r-')
    ax[3].set_title('Average FWHM VS Time')
    ax[3].set_xlabel('Time [seconds]')
    ax[3].set_ylabel('Average FWHM [pixels]')
    ax[3].set_xlim([np.min(time_array), np.max(time_array)])
    ax[3].set_ylim([0, np.max(FWHM_avg)*1.1])

    plt.ion()
    try:
        while True:
            img = take_frame()
            if img.size == 0: #check if img is empty
                continue
            if np.max(img) == np.min(img): #check if img has the same values for all pixels
                continue
            frame_time = time.time() #time at which frame was taken

            x_cen, y_cen = find_centroid_iter(img, cen_tol=5, maxiter=10, do_plot=False)
            cropped_img, crop_radius = crop_image_for_fit(img, x_cen, y_cen, radius_scale=1)
            if cropped_img.size == 0:
                continue
            if 0 in cropped_img.shape:
                continue
            h, w = cropped_img.shape

            params, params_uncert, params_dict, params_uncert_dict, fit_fcn = fit_gaussian(cropped_img)
            FWHM_x = gaussian_FWHM(params_dict['sigma_x'])
            FWHM_y = gaussian_FWHM(params_dict['sigma_y'])
            FWHM_avg = (FWHM_x + FWHM_y)/2
            x = np.arange(cropped_img.shape[1])
            y = np.arange(cropped_img.shape[0])
            X, Y = np.meshgrid(x, y)
            Z = gaussian(X, Y, *params)

            #Update original image
            im0.set_data(img)
            im0.set_clim(np.min(img), np.max(img))
            rect.set_xy([x_cen-crop_radius, y_cen-crop_radius])
            rect.set_height(crop_radius*2)
            rect.set_width(crop_radius*2)

            #Update cropped region
            im1.set_data(cropped_img)
            im1.set_clim(np.min(cropped_img), np.max(cropped_img))

            #Update fitted Gaussian
            im2.set_data(Z)
            im2.set_clim(np.min(Z), np.max(Z))
            plot_text_x = "FWHM$_{x}$ = " + "{:.3f} pixels\n".format(FWHM_x)
            plot_text_y = "FWHM$_{y}$ = " + "{:.3f} pixels\n".format(FWHM_y)
            tx.set_text(plot_text_x)
            ty.set_text(plot_text_y)

            #Update time_array
            time_array = np.append(time_array, frame_time - start_time)
            time_array = time_array[1:]
            #Update FWHM array
            FWHM_array = np.append(FWHM_array, FWHM_avg)
            FWHM_array = FWHM_array[1:]
            #Update Time VS Average FWHM plot
            time_plot.set_data(time_array, FWHM_array)
            ax[3].set_xlim([np.min(time_array), np.max(time_array)])
            ax[3].set_ylim([0, np.max(FWHM_array*1.1)])

            plt.pause(0.05)
    except KeyboardInterrupt:
        pass

    plt.ioff()
    plt.show()

    camera_obj.done_camera()
    print("Done!")


import sys
input = sys.stdin.readline

def find(parent,x):
    if x!=parent[x]:
        parent[x] = find(parent,parent[x])
    return parent[x]

def union(parent,a,b):
    a = find(parent,a)
    b = find(parent,b)
    if a<b:parent[b] = a
    else:parent[a] = b

n,m = map(int,input().split())
edges,result = [],[]
parent = [i for i in range(n+1)]
for _ in range(m):
    a,b,c = map(int,input().split())
    edges.append((c,a,b))
edges.sort()

for cost,a,b in edges:
    if find(parent,a) != find(parent,b):
        union(parent,a,b)
        result.append(cost)
print(sum(result[:-1]))

백준/Gold/1647. 도시 분할 계획/도시 분할 계획.py


##
##

import logging
from typing import Union, List
from couchformation.aws.driver.base import CloudBase, AWSDriverError, EmptyResultSet
from couchformation.aws.driver.constants import AWSTagStruct, AWSTag

logger = logging.getLogger('couchformation.aws.driver.gateway')
logger.addHandler(logging.NullHandler())
logging.getLogger("botocore").setLevel(logging.ERROR)
logging.getLogger("urllib3").setLevel(logging.ERROR)

class InternetGateway(CloudBase):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def list(self) -> List[dict]:
        ig_list = []
        igs = []
        extra_args = {}

        try:
            while True:
                result = self.ec2_client.describe_internet_gateways(**extra_args)
                igs.extend(result['InternetGateways'])
                if 'NextToken' not in result:
                    break
                extra_args['NextToken'] = result['NextToken']
        except Exception as err:
            raise AWSDriverError(f"error getting Internet Gateway list: {err}")

        for ig_entry in igs:
            ig_block = {'owner': ig_entry['OwnerId'],
                        'attachments': [a['VpcId'] for a in ig_entry['Attachments']],
                        'id': ig_entry['InternetGatewayId']}
            ig_list.append(ig_block)

        if len(ig_list) == 0:
            raise EmptyResultSet(f"no Internet Gateways found")
        else:
            return ig_list

    def create(self, name: str, vpc_id: str) -> str:
        ig_tag = [AWSTagStruct.build("internet-gateway").add(AWSTag("Name", name)).as_dict]
        try:
            result = self.ec2_client.create_internet_gateway(TagSpecifications=ig_tag)
            ig_id = result['InternetGateway']['InternetGatewayId']
            self.ec2_client.attach_internet_gateway(InternetGatewayId=ig_id, VpcId=vpc_id)
        except Exception as err:
            raise AWSDriverError(f"error creating Internet Gateway: {err}")

        return ig_id

    def delete(self, ig_id: str) -> None:
        try:
            ig_data = self.details(ig_id)
            if not ig_data:
                return
            for vpc_id in ig_data.get('attachments', []):
                self.ec2_client.detach_internet_gateway(InternetGatewayId=ig_id, VpcId=vpc_id)
            self.ec2_client.delete_internet_gateway(InternetGatewayId=ig_id)
        except Exception as err:
            raise AWSDriverError(f"error deleting Internet Gateway: {err}")

    def details(self, ig_id: str) -> Union[dict, None]:
        try:
            result = self.ec2_client.describe_internet_gateways(InternetGatewayIds=[ig_id])
            ig_entry = result['InternetGateways'][0]
            ig_block = {'owner': ig_entry['OwnerId'],
                        'attachments': [a['VpcId'] for a in ig_entry['Attachments']],
                        'id': ig_entry['InternetGatewayId']}
            return ig_block
        except (KeyError, IndexError):
            return None
        except Exception as err:
            raise AWSDriverError(f"error getting Internet Gateway details: {err}")


import pandas as pd
import numpy as np
from surprise import Dataset
from surprise import Reader
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from books.models import Book_similarities

from surprise import KNNBasic
import sklearn.metrics as metrics
from sklearn.neighbors import NearestNeighbors
from scipy.spatial.distance import correlation, cosine
import ipywidgets as widgets
from IPython.display import display, clear_output
from sklearn.metrics import pairwise_distances
from sklearn.metrics import mean_squared_error
from math import sqrt
import sys, os
from contextlib import contextmanager

class Command(BaseCommand):
    help = 'Closes the specified poll for voting'
    metric = 'cosine'
    k = 6
    sample_limit = 10000
    user_id = 183

    def handle(self, *args, **options):
        ratings = pd.read_csv('data/BX-Book-Ratings.csv', sep=';', error_bad_lines=False, encoding="latin-1")
        ratings.columns = ['userID', 'ISBN', 'bookRating']

        #Reduce the dataset size, take into account users who have rated at least 100 books
        #and books which have at least 100 ratings
        counts1 = ratings['userID'].value_counts()
        ratings_explicit = ratings[ratings['userID'].isin(counts1[counts1 >= 100].index)]
        counts = ratings_explicit['bookRating'].value_counts()
        ratings_explicit = ratings_explicit[ratings_explicit['bookRating'].isin(counts[counts >= 100].index)]

        #Generate a user-item ratings matrix from the ratings table.
        ratings_matrix = ratings_explicit.pivot(index='userID', columns='ISBN', values='bookRating')
        for col in ratings_matrix:
            ratings_matrix[col].fillna(0, inplace=True)
        self.calculateSimilarBooks(ratings_matrix)

    """
        Calculate similarities of books

        @param obj self           The pointer of class
    """
    def calculateSimilarBooks(self, ratings_matrix):
        counter = 0
        for i in range(ratings_matrix.shape[1]):
            item_id = str(ratings_matrix.columns[i])
            similarities, indices= self.findksimilaritems(item_id, ratings_matrix)
            for i in range(0, len(indices.flatten())):
                if (similarities[i] != 0):
                    index = indices.flatten()[i]
                    similar_item_id = str(ratings_matrix.columns[index])
                    self.saveSimilarBooks(item_id,similar_item_id)
                counter = counter + 1
                if (counter > self.sample_limit):
                    break

    """
        Find similarities bettween items

        @param obj self           The pointer of class
        @param int item_id        The id of item
        @param arr ratings_matrix The matrix of ratings
    """
    def findksimilaritems(self, item_id, ratings_matrix):
        similarities=[]
        indices=[]
        ratings = ratings_matrix.T
        loc = ratings.index.get_loc(item_id)
        model_knn = NearestNeighbors(metric = 'cosine', algorithm = 'brute')
        model_knn.fit(ratings)
        distances, indices = model_knn.kneighbors(ratings.iloc[loc, :].values.reshape(1, -1), n_neighbors = self.k + 1)
        similarities = 1-distances.flatten()

        return similarities,indices

    """
        Save user book predictions into db

        @param obj self       The pointer of class
        @param int item_id    The id of item
        @param int similar_id The similar id
    """
    def saveSimilarBooks(self, item_id, similar_id):
        book_similarities = Book_similarities()
        book_similarities.ISBN = item_id
        book_similarities.ISBN_similar = similar_id
        book_similarities.save()