"""
You must log in before proceeding with the code
Launch remote window. copy and paste the below command into command prompt window (cmd)
chrome.exe --remote-debugging-port=9222 --user-data-dir="C:\selenum\ChromeProfile)

"""
#All necessary imports from pip installs
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains

from datetime import date
from selenium.webdriver.common.keys import Keys
import win32com.client
from bs4 import BeautifulSoup
import pandas as pd
import re
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.styles import Font, Border, Side

# Configure Chrome options
chrome_driver = r'C:\chromedriver.exe'
options = Options()
options.add_argument('--headless')
options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
service = Service(chrome_driver)
driver = webdriver.Chrome(service=service, options=options)
print("Setup Successful")

# Open PIRATe Console Log --> User must manually login!!
ConsoleLog = 'https://mer1.jsc.nasa.gov/ConsoleLog/Default.aspx'
driver.get(ConsoleLog)
wait = WebDriverWait(driver, 10)
print("Parent window is: "+ driver.title)
parent = driver.current_window_handle

# Open Console Log via Login
iframe_element =driver.find_element(By.CSS_SELECTOR,'iframe[src="/ConsoleLog/Home.aspx"][class="resizable"][title="Console Log Frame"]')
driver.switch_to.frame(iframe_element)
driver.find_element("xpath","//*[@id='TM-A0']").click()
time.sleep(1)
driver.find_element("xpath","//*[@id='TM-A5']").click()
time.sleep(15)

# Switch working driver from MER Console Log - Default Page --> Console Log Printable Report
print("Switching drivers")
window_handles = driver.window_handles
for handle in window_handles:
    if(handle!=parent):
        driver.switch_to.window(handle)
        break

time.sleep(1)
page = driver.page_source
soup = BeautifulSoup(page, 'html.parser')
table = soup.find('table', id='ListGrid')

page = driver.page_source
soup = BeautifulSoup(page, 'html.parser')
table = soup.find('table', id='ListGrid')
df = pd.DataFrame()

if table:
    rows = table.find_all('tr', class_=['CLGridRow', 'CLGridAlternatingRow'])

    for row in rows:

        # Find and extract the elements containing the dates and bold/strong elements
 ######################################################################################################
 #                              TO CHANGE PER MONTH (must do every time):                             #
 #                                                                                #
 #      date_elements = row(text=re.compile(r'XX/\d{2}/\d{2}'))                                       #
 #      matching_dates = [date for date in date_elements if re.match(r'XX/\d{2}/\d{2}', date)]        #
 #                                                                                #
 #                                         == CHANGE 'X' ==                                           #
 ######################################################################################################

        date_elements = row(text=re.compile(r'06/\d{2}/\d{2}'))
        matching_dates = [date for date in date_elements if re.match(r'06/\d{2}/\d{2}', date)]

        bold_texts = row.find_all('span', class_='clBold')
        strong_texts = row.find_all('span', class_='clStrong')
        extracted_texts = [text.get_text(strip=True) for text in bold_texts + strong_texts]

        # Find bold bulleted text within the row
        bullets = row.find_all('li')
        extracted_bullets = []
        for bullet in bullets:
            bold_bullet = bullet.find_all('b')
            strong_bullet = bullet.find_all('strong')
            extracted_bullets += [bold_bullet.get_text(strip=True) for bold_bullet in bold_bullet]
            extracted_bullets += [strong_bullet.get_text(strip=True) for strong_bullet in strong_bullet]

        # Combine the extracted bold/strong texts and bold bulleted text
        extracted_texts += extracted_bullets

        # Print only the relevant month dates
        if date_elements:
            entryTime = [date_elements[0]]
            datesdf = pd.DataFrame(entryTime, columns=['dates'])
            bolddf = pd.DataFrame(extracted_texts, columns=['bold/strong text'])

            anything = pd.concat([datesdf, bolddf], axis=1)
            df = pd.concat([df, anything], axis=0).reset_index(drop=True)
else:
    print("No table found in the HTML.")

patterns = [r'OCR', r'FIT', r'AR', r'PAR', r'FITs', r'Event\b', r'Chit', r'Chits', r'Flight Note', r'FN', r'IFI', r'IFN', r'VIP', r'R&R', r'TIF']

CORE_Facilities = [r'CIR', r'Advanced Plant Habitat', r'Air Save Pump', r'BER', r'BEAM', r'BIOS', r'Cryo Chiller', r'Cyclops', r'ELC', r'ESP', r'EXPRESS', r'ExpRk', r'FIR', r'FRIDGE', r'Gallery Refrigerator', r'Gallery Refrigerator', r'Glacier', r'Glovebox Freezer', r'Iceberg', r'ISS Portable Glovebag', r'Life Sciences Glovebox', r'LSG', r'MAMS', r'MELFI', r'MERLIN', r'MQ-Warmchain', r'MSG', r'MSRR', r'NanoRacks Airlock', r'Other Cold Stow', r'POLAR', r'Rodent Research', r'SAMS', r'Wetlab1', r'Wetlab2', r'WORF', r'Axiom', r'Ax']

CORE_Facilities_groups = {
    'CIR': ['CIR'],
    'Advanced Plant Habitat':['Advanced Plant Habitat', 'Plant Habitat', 'PlantHabitat'],
    'Air Save Pump':['Air Save Pump'],
    'BER':['BER'],
    'BEAM':['BEAM'],
    'BIOS':['BIOS'],
    'Cryo Chiller':['Cryo Chiller'],
    'Cyclops':['Cyclops'],
    'ELC':['ELC','ESP'],
    'EXPRESS':['EXPRESS', 'ExpRk', 'ExpRk4', 'EXPRESS Rack'],
    'FIR':['FIR'],
    'FRIDGE':['FRIDGE'],
    'Galley Refrigerator':['Galley Refrigerator','Galley Freezer'],
    'Glacier':['Glacier'],
    'Glovebox Freezer':['Glacier Freezer'],
    'Iceberg':['Iceberg'],
    'ISS Portable Glovebag': ['ISS Portable Glovebag'],
    'Life Sciences Glovebox (LSG)':['Life Sciences Glovebox', 'LSG'],
    'MAMS':['MAMS'],
    'MELFI':['MELFI'],
    'MERLIN':['MERLIN'],
    'MQ-Warmchain':['MQ-Warmchain'],
    'MSG':['MSG'],
    'MSSR':['MSSR'],
    'NanoRacks Airlock':['NanoRacks Airlock'],
    'Other Cold Stow':['Other Cold Stow'],
    'POLAR':['POLAR'],
    'Rodent Research':['Rodent Research'],
    'SAMS':['SAMS'],
    'Wetlab1':['Wetlab1'],
    'Wetlab2':['Wetlab2'],
    'WORF':['WORF'],
    'Axiom Module':['Axiom Module','AX','Axiom','Ax'],
}

pattern_groups = {
    'OCR': ['OCR'],
    'PAR': ['PAR', 'SW'],
    'AR':['AR'],
    'Chits': ['Chit', 'Chits'],
    'Events': [r'Event\b', 'FIT', 'FITs', 'R&R', 'TIF'],
    'FN': ['Flight Note', 'FN'],
    'IFI': ['IFI'],
    'IFN': ['IFN']
}
# Filter the dataframe based on the patterns
filtered_df = df[df['bold/strong text'].str.contains('|'.join(patterns), regex=True, na=False)]
filtered_df.sort_values(by=['bold/strong text'])

# Extract miscellaneous data before filtering with patterns
miscellaneous_data = df[~df['bold/strong text'].str.contains('|'.join(patterns), regex=True, na=False)]
miscellaneous_data = miscellaneous_data[~miscellaneous_data['bold/strong text'].str.contains(r'10:00am MER Tagup|Off Console|Checkout With MER Mgr|10:00 MER Tag-up|Events Since Last Shift|On Console|10:00 MER Tagup|Checkout with MER Manager')]
miscellaneous_data.sort_values('bold/strong text')

# Initialize a set to keep track of assigned texts
assigned_rows = set()

# Group the patterns in sheets based on custom groups
grouped_data = {'Miscellaneous': miscellaneous_data}

for group_name, group_patterns in pattern_groups.items():
    group_data = filtered_df[filtered_df['bold/strong text'].apply(lambda x: any(pattern in x for pattern in group_patterns))].copy()
    for facility in CORE_Facilities_groups.keys():
        group_data.loc[:, facility] = group_data['bold/strong text'].str.contains('|'.join(CORE_Facilities_groups[facility]), regex=True, na=False).astype(int)

    grouped_data[group_name] = group_data

# Export the grouped DataFrames to separate sheets in the Excel file
with pd.ExcelWriter("Console Log Output.xlsx") as writer:
    df.to_excel(writer, sheet_name='All Bold Texts', index=False)

    for group_name, data in grouped_data.items():
        # Exclude pattern columns if they exist in the DataFrame
        patterns_to_drop = list(set(patterns) & set(data.columns))
        data.drop(patterns_to_drop, axis=1, inplace=True)

        # Check if the sheet name is "All Bold Texts" and remove duplicates accordingly
        if group_name == 'All Bold Texts':
            data.to_excel(writer, sheet_name=group_name, index=False)
        else:
            # Remove duplicate rows based on 'dates' and 'bold/strong text' columns
            data.drop_duplicates(subset=['dates', 'bold/strong text'], inplace=True)
            data.to_excel(writer, sheet_name=group_name, index=False)

import pandas as pd

# Organize Excel File Further
excel_file = pd.ExcelFile("Console Log Output.xlsx")
organized_data = {}

for sheet_name in excel_file.sheet_names:
    data = pd.read_excel(excel_file, sheet_name=sheet_name)
    sorted_data = data.sort_values('bold/strong text')

    if sheet_name == 'AR':
        sorted_data = sorted_data[~sorted_data['bold/strong text'].str.contains(r'PAR|OCR|Chit|Flight Note|FN|POLAR|ARM')]

    if sheet_name == 'Chit':
        sorted_data['pattern'] = sorted_data['bold/strong text'].str.extract(r'(Chit\s\d{5})', expand=False)
        sorted_data = sorted_data.drop_duplicates(subset='pattern', keep='first')
        sorted_data = sorted_data.drop('pattern', axis=1)

    if sheet_name == 'IFI':
        sorted_data = sorted_data[~sorted_data['bold/strong text'].str.contains(r'Chit')]

    if sheet_name == 'PAR':
        sorted_data = sorted_data[~sorted_data['bold/strong text'].str.contains(r'OCR')]

    if sheet_name == 'OCR':
        sorted_data = sorted_data[~sorted_data['bold/strong text'].str.contains(r'IFN')]

    organized_data[sheet_name] = sorted_data

with pd.ExcelWriter("Console Log Output Organized.xlsx") as writer:
    for sheet_name, data in organized_data.items():
        data.to_excel(writer, sheet_name=sheet_name, index=False)

# Define a list of sheets to process
sheets_to_process = ['OCR', 'PAR', 'AR', 'Chits', 'Events', 'FN', 'IFI', 'IFN']
organized_data = pd.read_excel("Console Log Output Organized.xlsx", sheet_name=None)

# Load the workbook
book = load_workbook("Console Log Output Organized.xlsx")

def filter_numeric_values(value):
    return value if isinstance(value, (int, float)) else 0

# Loop through the sheets in the workbook
for sheet_name in book.sheetnames:
    if sheet_name not in sheets_to_process:
        continue

    # Get the DataFrame for the current sheet
    data = organized_data[sheet_name]

    # Check if the sheet is present in the organized_data and contains data
    if data is not None and not data.empty:
        # Calculate the row-wise sum and add it to the last column
        data['Sum'] = data.iloc[:,1:].applymap(filter_numeric_values).sum(axis=1)

        # Write the updated data back to the sheet
        worksheet = book[sheet_name]
        column_number = worksheet.max_column + 1

        for row_num, row in data.iterrows():
            sum_value = row['Sum']
            if sum_value > 0:
                final_value = 0
            else:
                final_value = 1
            worksheet[f'{get_column_letter(column_number)}{row_num + 2}'] = final_value

        worksheet.cell(row=1, column=column_number).value='Other or Unnamed Payload Facilities'

# Save the changes to the workbook
book.save("Console Log Output Organized.xlsx")

# Load the workbook to add formulas
workbook = load_workbook("Console Log Output Organized.xlsx")

# Reverse the CORE_Facilities_groups dictionary to map payload facilities to group names
group_name_to_facilities = {}
for group_name, facilities in CORE_Facilities_groups.items():
    facilities_string = '|'.join(facilities)
    group_name_to_facilities[facilities_string] = group_name

# Loop through the sheets in the workbook
for sheet_name in workbook.sheetnames:
    if sheet_name not in organized_data:
        continue

    data = organized_data[sheet_name]
    data.columns = data.columns.astype(str)
    data.rename(columns=group_name_to_facilities, inplace=True)
    worksheet = workbook[sheet_name]
    row_number = worksheet.max_row + 1

    # Write the formula to the last row in each column (C to AJ) for Column Sum
    for col_num in range(3, 38):  # ASCII values for 'C' to 'AJ' are 67 to 90, respectively
        column_letter = get_column_letter(col_num-1)
        if worksheet.max_row == 1:
            continue
        else:
            formula = f'=SUBTOTAL(109, {column_letter}2:{column_letter}{row_number - 1})' # 109 for SUM function ignoring hidden rows
            worksheet[f'{column_letter}{row_number}'] = formula

workbook.save("Console Log Output Organized.xlsx")

print('Sums complete')

# Define a list of sheets to process
sheets_to_process = ['OCR', 'PAR', 'AR', 'Chits', 'Events', 'FN', 'IFI', 'IFN']

# Read the organized data from the existing Excel file
organized_data = pd.read_excel("Console Log Output Organized.xlsx", sheet_name=None)

# Create a new summary DataFrame
summary_data = pd.DataFrame(index=list(CORE_Facilities_groups.keys()) + ["Other or Unnamed Payload Facilities"], columns=sheets_to_process)

# Fill in the summary DataFrame with counts from each sheet
for sheet_name in sheets_to_process:
    if sheet_name in organized_data:
        data = organized_data[sheet_name]
        for facility in CORE_Facilities_groups.keys():
            summary_data.at[facility, sheet_name] = data[facility].sum()
        # Use the negative column index -1 to access the last column
        summary_data.at["Other or Unnamed Payload Facilities", sheet_name] = data.iloc[:, -1].sum()

# Load the existing Excel file
book = load_workbook("Console Log Output Organized.xlsx")

# Add a new sheet for the summary data
summary_sheet_name = 'Summary'
book.create_sheet(title=summary_sheet_name, index=len(book.sheetnames))

# Write the summary data to the new sheet
summary_sheet = book[summary_sheet_name]
for row in dataframe_to_rows(summary_data, index=True, header=True):
    summary_sheet.append(row)

# Apply formatting to the summary sheet
header_font = Font(bold=True)
thin_border = Border(left=Side(style='thin'), right=Side(style='thin'), top=Side(style='thin'), bottom=Side(style='thin'))

for row in summary_sheet.iter_rows(min_row=1, max_row=1):
    for cell in row:
        cell.font = header_font
        cell.border = thin_border

# Get the row number for the sum formula
row_number = len(summary_data) + 4
feu_row = len(summary_data) + 5

# List of Multipliers
multipliers = [0.5, 0.2, 0.3, 0.3, 1, 0.3, 0.5, 0.5]

# Create overall sum for each type of note (AR, PAR, etc)
for col_num in range(2, 10):  # ASCII values for 'C' to 'I'
    column_letter = get_column_letter(col_num)
    formula = f'=SUM({column_letter}2:{column_letter}{row_number - 1})'
    summary_sheet[f'{column_letter}{row_number}'] = formula

    # Calculate total feu for each column
    total_feu_formula = f'={column_letter}{row_number} * {multipliers[col_num-2]}'
    summary_sheet[f'{column_letter}{feu_row}'] = total_feu_formula

    total_metric = f'=SUM(B{feu_row}:I{feu_row})'
    summary_sheet[f'J{feu_row}'] = total_metric

# Save the updated Excel file
book.save("Console Log Output Organized.xlsx")


#/usr/lib/python2.7
import sys
import socket
import threading

def server_loop(local_host, local_port, remote_host,
                remote_port, receive_first):
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
        server.bind((local_host, local_port))
    except:
        print ("""Failed to listen {0}:{1}!!
              Please try again!""".format(local_host, local_port))
        sys.exit(0)
    print ("-*- Listen on {0:16}:{1:5}".format(local_host, local_port))
    server.listen(5)

    while True:
        cli_socket, cli_addr = server.accept()
        print ("[==>]   Received incoming connection from {0:16}:{1:5}".format(cli_addr[0],
                                                                               cli_addr[1]))
        proxy_thread = threading.Thread(target = proxy_handler, args = (cli_socket, remote_host,
                                                                        remote_port, receive_first))
        proxy_thread.start()

def main():
    if len(sys.argv[1:]) != 5:
        print ("Usage: ./proxy.py [local_host] [local_port] [remote_host] [remote_port] [receive_first]")
        sys.exit(0)

    local_host = sys.argv[1]
    local_port = int(sys.argv[2])
    remote_host = sys.argv[3]
    remote_port = int(sys.argv[4])
    receive_first = sys.argv[5]

    if "True" or "true" in receive_first:
        receive_first = True
    else:
        receive_first = False

    server_loop(local_host, local_port, remote_host, remote_port, receive_first)

def proxy_handler(client_socket, remote_host, remote_port, receive_first):

    remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    remote_socket.connect((remote_host, remote_port))

    if receive_first:
        remote_buffer = receive_from(remote_socket)
        hexdump(remote_buffer)
        remote_buffer = response_to_handler(remote_buffer)
        if len(remote_buffer):
            print("[<==] Sending {0} bytes to local_host".format(len(remote_buffer)))
            client_socket.send(remote_buffer)

    while True:
        localbuffer = receive_from(client_socket)
        if len(localbuffer):
            print ("[==>] Received {0} bytes from local_host.".format(len(localbuffer)))
            hexdump(localbuffer)
            localbuffer = request_handler(localbuffer)
            remote_socket.send(localbuffer)
            print ("[==>] Sent to remote")

        remote_buffer = receive_from(remote_socket)

main()


import sys
import time

from random import randint

import zmq

def main(url=None):
    ctx = zmq.Context.instance()
    subscriber = ctx.socket(zmq.SUB)
    if url is None:
        url = "tcp://localhost:5557"
    subscriber.connect(url)

    publisher=ctx.socket(zmq.PUB)
    publisher.bind("tcp://localhost:5558")

    subscription = b"%03d" % 1
    print(subscription)
    subscriber.setsockopt(zmq.SUBSCRIBE, subscription)

    while True:
        topic, data = subscriber.recv_multipart()
        print(topic)
        assert topic == subscription
        print (data)
        time.sleep(1)
        publisher.send_multipart([
                b"%03d" % 2,
                b"Off with his head!",
            ])
if _name_ == '_main_':
    main(sys.argv[1] if len(sys.argv) > 1 else None)


import time, pytest, inspect
from utils import *
from PIL import Image

def test_overlay_at_start(run_brave, create_config_file):
    set_up_overlay_at_start(run_brave, create_config_file)
    assert_overlays([{'id': 1, 'uid': 'overlay1'}])

    add_overlay({'type': 'text', 'source': 'mixer1', 'text': 'Overlay #1', 'visible': True})
    time.sleep(1)
    assert_overlays([{'id': 1, 'source': 'mixer1', 'visible': True},
                     {'id': 2, 'source': 'mixer1', 'visible': True}])

    # Try adding one that's not visible:
    add_overlay({'type': 'text', 'source': 'mixer1', 'text': 'Overlay #2', 'visible': False})
    time.sleep(1)
    assert_overlays([{'id': 1, 'visible': True},
                     {'id': 2, 'visible': True},
                     {'id': 3, 'visible': False}])

    # Try changing visible flag
    update_overlay(1, {'visible': False})
    update_overlay(3, {'visible': True})
    time.sleep(1)
    assert_overlays([{'id': 1, 'visible': False},
                     {'id': 2, 'visible': True},
                     {'id': 3, 'visible': True}])

    subtest_delete_overlay()
    subtest_add_overlay_without_source()
    subtest_make_overlay_without_source_visible()

def subtest_delete_overlay():
    delete_overlay(3)
    time.sleep(1)
    assert_overlays([{'id': 1, 'visible': False},
                     {'id': 2, 'visible': True}])

def subtest_add_overlay_without_source():
    add_overlay({'type': 'text', 'text': 'Overlay #3b', 'visible': False})
    time.sleep(1)
    assert_overlays([{'id': 1, 'visible': False, 'source': 'mixer1'},
                     {'id': 2, 'visible': True, 'source': 'mixer1'},
                     {'id': 3, 'visible': False, 'source': None}])

def subtest_make_overlay_without_source_visible():
    # Cant't make visible if no source - so will return with a 400
    update_overlay(3, {'visible': True}, status_code=400)
    time.sleep(1)
    assert_overlays([{'id': 1, 'visible': False, 'source': 'mixer1'},
                     {'id': 2, 'visible': True, 'source': 'mixer1'},
                     {'id': 3, 'visible': False, 'source': None}])

def set_up_overlay_at_start(run_brave, create_config_file):
    output_video_location = create_output_video_location()

    config = {
    'mixers': [
        {}
    ],
    'overlays': [
        {'type': 'text', 'source': 'mixer1', 'text': 'Overlay #1', 'visible': True}
    ],
    'outputs': [
        # {'type': 'local'} # good for debugging
        {'type': 'local'}
    ]
    }
    config_file = create_config_file(config)
    run_brave(config_file.name)
    time.sleep(2)
    check_brave_is_running()


import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder

dataset = pd.read_csv('iris.csv')
encoder = LabelEncoder()
X = dataset.iloc[:, 1:5].values
y = dataset.iloc[:, -1].values

y1 = encoder.fit_transform(y)

Y = pd.get_dummies(y1).values

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=0, stratify=Y)
np.random.seed(42)

def error_square(out,w):
    res=0
    for i in range(len(out)):
        res+=0.5*(w[i]-out[i])**2
    return res
class Layer:

    def __init__(self):

        pass

    def calculateError(self,out,w):
        return w-out
    def derivateActivation(self,inp):
        return inp
    def forward(self, inp):
        # a dummy layer returns the input
        return inp

    def backward(self, inp, grad_outp):
        pass

class ReLU(Layer):
    def __init__(self):
        pass
    def derivateActivation(self,inp):
        res=np.copy(inp)
        for i in range(len(inp)):
            if inp[i] < 0:
                res[i]=0
            else:
                res[i]=1
        return res

        return 1
    def forward(self, inp):
        return np.maximum(0, inp)

    def backward(self, inp, grad_outp):
        grad_outp1 = np.ndarray(shape=(len(inp),))
        for i in range(len(grad_outp)):
            grad_outp1[i] = np.sum(grad_outp[i])
        return grad_outp1*self.derivateActivation(inp)

class Sigmoid(Layer):
    def __init__(self):
        pass

    def derivateActivation(self,inp):
        return self.forward(inp)*(1-self.forward(inp))

    def forward(self, inp):
        return 1 / (1 + np.exp(-inp))

    def backward(self, inp, grad_outp):
        grad_outp1=np.ndarray(shape=(len(inp),))
        for i in range(len(grad_outp)):
            grad_outp1[i]=np.sum(grad_outp[i])
        return grad_outp1*self.derivateActivation(inp)

class Dense(Layer):
    def __init__(self, inp_units, outp_units, learning_rate=0.01):
        self.learning_rate = learning_rate
        self.weights = np.random.normal(
            loc=0.0, scale=np.sqrt(2 / (inp_units + outp_units)),
            size=(inp_units, outp_units))
        self.neuron_count = outp_units
        self.updatesWeights = np.copy(self.weights)
        self.biases = np.zeros(outp_units)

    def update_weights(self):
        for x in range(len(self.weights)):
            for y in range(self.get_neuron_count()):
                self.weights[x][y]+=self.updatesWeights[x][y]

    def get_neuron_count(self):
        return self.neuron_count

    def forward(self, inp):
        return np.dot(inp, self.weights) + self.biases

    def backward(self, inp, grad_outp):
        for i in range(len(inp)):
            for x in range(self.get_neuron_count()):
                self.updatesWeights[i][x]=self.learning_rate*grad_outp[x]*inp[i]
        return self.weights*grad_outp

class MLP():
    def __init__(self):
        self.layers = []

    def add_layer(self, neuron_count, inp_shape=None, activation='ReLU'):
        if len(self.layers) == 0 and inp_shape is None:
            raise ValueError("Must defined input shape for first layer")

        if inp_shape is None:
            inp_shape = self.layers[-2].get_neuron_count()

        self.layers.append(Dense(inp_shape, neuron_count))
        if activation == 'sigmoid':
            self.layers.append(Sigmoid())
        elif activation == 'ReLU':
            self.layers.append(ReLU())
        else:
            raise ValueError("Unknown activation function", activation)

    def forward(self, X):
        activations = []
        layer_input = X
        for l in self.layers:

            activations.append(l.forward(layer_input))
            layer_input = activations[-1]

        return activations

    def predict(self, X):
        logits = self.forward(X)[-1]
        return logits.argmax(axis=-1)

    def fit(self, X, Y,epohs=1000):
        good = 0
        all = 0
        errors = []
        for epoch in range(epohs+1):
            for i in zip(X,Y):
                outputs=self.forward(i[0])
                error=self.layers[-1].calculateError(outputs[-1],i[1])
                error=error*self.layers[-1].derivateActivation(outputs[-2])
                error=self.layers[-2].backward(outputs[-3],error)
                if self.predict(i[0])==np.argmax(i[1],axis=-1):
                    good+=1
                all+=1
                errors.append(error_square(outputs[-1],i[1]))
                for x in range(-3,-len(self.layers),-2):
                    error=self.layers[x].backward(outputs[x-1],error)
                    if x==-len(self.layers)+1:
                        error = self.layers[x - 1].backward(i[0], error)
                    else:
                        error=self.layers[x-1].backward(outputs[x-2],error)

                for layer in self.layers:
                    if isinstance(layer,Dense):
                        layer.update_weights()
            if epoch%50==0 and epoch!=0:
                print("Epoch {}: Accuracy={},Error={}".format(epoch,round(good/all,4),sum(errors)/len(errors)))
                good=0
                all=0
                errors=[]
        print("End of training.")

if __name__ == '__main__':
    network = MLP()
    network.add_layer(6, 4, activation='sigmoid')
    network.add_layer(7, activation='sigmoid')
    network.add_layer(3, activation='sigmoid')
    network.fit(X_train,y_train)
    print("Tests:")
    y_pred = network.predict(X_test)

    y_test_class = np.argmax(y_test, axis=1)

    from sklearn.metrics import classification_report,confusion_matrix
    print("Confusion matrix:")
    print(confusion_matrix(y_test_class, y_pred))
    print("Classification report:")
    print(classification_report(y_test_class, y_pred))


import socket
import queue
import threading
import os
import time
import cprotocol as prot
from Encryption import Defi

class ClientCom:

    def __init__(self, serverIp, serverPort, q, file = False, key = None):
        '''

        :param file(optional): "true" if this network is to recv file and "false" otherwise
        :param serverIp:ip of the server
        :param serverPort: port to talk with the server
        :param q: queue that for trnsger data between the network to the main client
        :param key(optional): aes key
        '''
        self.file = file
        self.serverIp = serverIp
        self.serverPort = serverPort
        self.q = q
        self.soc = socket.socket()

        if not file and not key:
            # need to switch keys with the server
            threading.Thread(target=self.switch_keys, daemon=True).start()
        elif file:
            # already have a key -> just connect without switch keys
            self.connect()
        else:
            self.key = key
            self.connect()
            threading.Thread(target=self.recv_file, daemon=True).start()

    def switch_keys(self):
        '''

        :return:switch keys with the server
        '''

        # create defi key
        defi = Defi()
        my_publish = defi.publish()

        try:
            # connect to the server
            self.soc.connect((self.serverIp, self.serverPort))
            # switch keys with the server
            server_publish = int(self.soc.recv(5).decode())
            self.soc.send(str(my_publish).encode())
        except Exception as e:
            print(f'in switch_keys - {str(e)}')

        else:
            key_str = str(defi.compute_secret(server_publish))
            self.q.put(f'key-{key_str}')
            # start reciving messages
            threading.Thread(target=self.__recv_msg__(), daemon=True).start()

    def connect(self):
        '''

        :return:connect to the server
        '''
        try:
            self.soc.connect((self.serverIp, self.serverPort))
        except Exception as e:
            print(f'in connect - {str(e)}')
            #  notify the logic that the server closed
            self.q.put('disconnect')
            self.soc.close()
            exit()

    def __recv_msg__(self):
        '''

        :return:recive msg from the server and put the msg in q
        '''
        while True:
            try:
                msg_len = int(self.soc.recv(3).decode())
                msg = self.soc.recv(msg_len)
            except Exception as e:
                print(f'in recv msg  - {str(e)}')
                # notify the logic that the server closed
                self.q.put('disconnect')
                self.soc.close()
                exit()
            else:
                self.q.put(msg)

    def send_msg(self, msg):
        '''

        :param msg: string
        :return: sends the msg to the server
        '''
        if type(msg) == str:
            msg = msg.encode()
        try:
            self.soc.send(str(len(msg)).zfill(3).encode())
            self.soc.send(msg)
        except Exception as e:
            print(f'in send msg - {str(e)}')
            self.soc.close()

    def send_file(self, file_path, server_path, file_name):
        '''

        :param file_path: path for file
        :param server_path: path to put in the server
        :param file_name: name of the file
        :return: sends the data to the server
        '''
        time.sleep(0.1)

        with open(file_path, 'rb') as f:
            # data of the file
            data = f.read()

        msg_after_protocol = prot.create_upload_file_msg(server_path, len(data), file_name)
        total_msg = str(len(msg_after_protocol)).zfill(3) + msg_after_protocol

        try:
            self.soc.send(total_msg.encode())
            self.soc.send(data)
        except Exception as e:
            print(f'in send file - {str(e)}')
            self.soc.close()

    def recv_file(self, file_len, file_name, key, path = None):
        '''

        :return:recv file from the server, download the file to downloads and add msg to q when finish recive
        '''
        file_data = bytearray()
        file_len = int(file_len)
        # recv all the data in blocks
        while len(file_data) < file_len:
            size = file_len - len(file_data)
            try:
                if size >= 1024:
                    file_data.extend(self.soc.recv(1024))
                else:
                    file_data.extend(self.soc.recv(size))
                    break
            except Exception as e:
                print(f'in recv file - {str(e)}')
                self.q.put('no')
                self.soc.close()
                file_data = None
                break
        # if recived the file -> put it in downloads
        if file_data is not None:
            if path is None:
                path = os.path.expanduser('~/Downloads')
            with open(f'{path}\\{file_name}', 'wb') as f:
                f.write(file_data)

            # decrypt the file recived
            key.decrypt_file(f'{path}\\{file_name}')
            self.q.put('ok')

        self.soc.close()


import sys
from PyQt5.QtWidgets import *

class Uygulama(QWidget):

    def __init__(self):
        super().__init__()
        self.userinterface()
        self.sayac_1 = 0
        self.sayac_2 = 0

    def userinterface(self):
        self.label_1 = QLabel("1. Butona Hiç Tıklanmadı")
        self.buton_1 = QPushButton("1. Buton")
        self.buton_1.clicked.connect(self.buton_1_tiklandi)
        vbox_1 = QVBoxLayout()
        vbox_1.addWidget(self.buton_1)
        vbox_1.addWidget(self.label_1)
        vbox_1.addStretch()

        self.label_2 = QLabel("2. Butona Hiç Tıklanmadı")
        self.buton_2 = QPushButton("2. Buton")
        self.buton_2.clicked.connect(self.buton_2_tiklandi)
        vbox_2 = QVBoxLayout()
        vbox_2.addWidget(self.buton_2)
        vbox_2.addWidget(self.label_2)
        vbox_2.addStretch()

        mainbox = QHBoxLayout()
        mainbox.addStretch()
        mainbox.addLayout(vbox_1)
        mainbox.addLayout(vbox_2)
        mainbox.addStretch()

        self.setLayout(mainbox)
        self.show()

    def buton_1_tiklandi(self):
        self.sayac_1 += 1
        self.label_1.setText( "1. butona %s kez tıklandı" % str(self.sayac_1) )

    def buton_2_tiklandi(self):
        self.sayac_2 += 1
        self.label_2.setText( "2. butona {} kez tıklandı".format(self.sayac_2) )

app = QApplication(sys.argv)
uygulama = Uygulama()
sys.exit(app.exec_())


import collections
import contextlib
import re

import mock
from oslo.vmware import exceptions as vexc
from oslo.vmware import pbm

from nova import context
from nova import exception
from nova.network import model as network_model
from nova.openstack.common import uuidutils
from nova import test
from nova.tests.unit import fake_instance
from nova.tests.unit.virt.vmwareapi import fake
from nova.tests.unit.virt.vmwareapi import stubs
from nova.virt.vmwareapi import driver
from nova.virt.vmwareapi import ds_util
from nova.virt.vmwareapi import vm_util

class partialObject(object):
    def __init__(self, path='fake-path'):
        self.path = path
        self.fault = fake.DataObject()

class VMwareVMUtilTestCase(test.NoDBTestCase):
    def setUp(self):
        super(VMwareVMUtilTestCase, self).setUp()
        fake.reset()
        stubs.set_stubs(self.stubs)
        vm_util.vm_refs_cache_reset()

    def _test_get_stats_from_cluster(self, connection_state="connected",
                                     maintenance_mode=False):
        ManagedObjectRefs = [fake.ManagedObjectReference("host1",
                                                         "HostSystem"),
                             fake.ManagedObjectReference("host2",
                                                         "HostSystem")]
        hosts = fake._convert_to_array_of_mor(ManagedObjectRefs)
        respool = fake.ManagedObjectReference("resgroup-11", "ResourcePool")
        prop_dict = {'host': hosts, 'resourcePool': respool}

        hardware = fake.DataObject()
        hardware.numCpuCores = 8
        hardware.numCpuThreads = 16
        hardware.vendor = "Intel"
        hardware.cpuModel = "Intel(R) Xeon(R)"

        runtime_host_1 = fake.DataObject()
        runtime_host_1.connectionState = "connected"
        runtime_host_1.inMaintenanceMode = False

        runtime_host_2 = fake.DataObject()
        runtime_host_2.connectionState = connection_state
        runtime_host_2.inMaintenanceMode = maintenance_mode

        prop_list_host_1 = [fake.Prop(name="hardware_summary", val=hardware),
                            fake.Prop(name="runtime_summary",
                                      val=runtime_host_1)]
        prop_list_host_2 = [fake.Prop(name="hardware_summary", val=hardware),
                            fake.Prop(name="runtime_summary",
                                      val=runtime_host_2)]

        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.ObjectContent("prop_list_host1",
                                                   prop_list_host_1))
        fake_objects.add_object(fake.ObjectContent("prop_list_host1",
                                                   prop_list_host_2))

        respool_resource_usage = fake.DataObject()
        respool_resource_usage.maxUsage = 5368709120
        respool_resource_usage.overallUsage = 2147483648

        def fake_call_method(*args):
            if "get_dynamic_properties" in args:
                return prop_dict
            elif "get_properties_for_a_collection_of_objects" in args:
                return fake_objects
            else:
                return respool_resource_usage

        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', fake_call_method):
            result = vm_util.get_stats_from_cluster(session, "cluster1")
            mem_info = {}
            if connection_state == "connected" and not maintenance_mode:
                vcpus = 32
            else:
                vcpus = 16
            mem_info['total'] = 5120
            mem_info['free'] = 3072
            expected_stats = {'vcpus': vcpus, 'mem': mem_info}
            self.assertEqual(expected_stats, result)

    def test_get_stats_from_cluster_hosts_connected_and_active(self):
        self._test_get_stats_from_cluster()

    def test_get_stats_from_cluster_hosts_disconnected_and_active(self):
        self._test_get_stats_from_cluster(connection_state="disconnected")

    def test_get_stats_from_cluster_hosts_connected_and_maintenance(self):
        self._test_get_stats_from_cluster(maintenance_mode=True)

    def test_get_host_ref_no_hosts_in_cluster(self):
        self.assertRaises(exception.NoValidHost,
                          vm_util.get_host_ref,
                          fake.FakeObjectRetrievalSession(""), 'fake_cluster')

    def test_get_resize_spec(self):
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': 'bda5fb9e-b347-40e8-8256-42397848cb00',
                         'vcpus': 2, 'memory_mb': 2048}
        result = vm_util.get_vm_resize_spec(fake.FakeFactory(),
                                            fake_instance)
        expected = """{'memoryMB': 2048,
                       'numCPUs': 2,
                       'obj_name': 'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_cdrom_attach_config_spec(self):

        result = vm_util.get_cdrom_attach_config_spec(fake.FakeFactory(),
                                             fake.Datastore(),
                                             "/tmp/foo.iso",
                                             200, 0)
        expected = """{
    'deviceChange': [
        {
            'device': {
                'connectable': {
                    'allowGuestControl': False,
                    'startConnected': True,
                    'connected': True,
                    'obj_name': 'ns0: VirtualDeviceConnectInfo'
                },
                'backing': {
                    'datastore': {
                        "summary.maintenanceMode": "normal",
                        "summary.type": "VMFS",
                        "summary.accessible":true,
                        "summary.name": "fake-ds",
                        "summary.capacity": 1099511627776,
                        "summary.freeSpace": 536870912000,
                        "browser": ""
                    },
                    'fileName': '/tmp/foo.iso',
                    'obj_name': 'ns0: VirtualCdromIsoBackingInfo'
                },
                'controllerKey': 200,
                'unitNumber': 0,
                'key': -1,
                'obj_name': 'ns0: VirtualCdrom'
            },
            'operation': 'add',
            'obj_name': 'ns0: VirtualDeviceConfigSpec'
        }
    ],
    'obj_name': 'ns0: VirtualMachineConfigSpec'
}
"""

        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_lsilogic_controller_spec(self):
        # Test controller spec returned for lsiLogic sas adapter type
        config_spec = vm_util.create_controller_spec(fake.FakeFactory(), -101,
                          adapter_type="lsiLogicsas")
        self.assertEqual("ns0:VirtualLsiLogicSASController",
                         config_spec.device.obj_name)

    def test_paravirtual_controller_spec(self):
        # Test controller spec returned for paraVirtual adapter type
        config_spec = vm_util.create_controller_spec(fake.FakeFactory(), -101,
                          adapter_type="paraVirtual")
        self.assertEqual("ns0:ParaVirtualSCSIController",
                         config_spec.device.obj_name)

    def _vmdk_path_and_adapter_type_devices(self, filename, parent=None):
        # Test the adapter_type returned for a lsiLogic sas controller
        controller_key = 1000
        disk = fake.VirtualDisk()
        disk.controllerKey = controller_key
        disk_backing = fake.VirtualDiskFlatVer2BackingInfo()
        disk_backing.fileName = filename
        disk.capacityInBytes = 1024
        if parent:
            disk_backing.parent = parent
        disk.backing = disk_backing
        controller = fake.VirtualLsiLogicSASController()
        controller.key = controller_key
        devices = [disk, controller]
        return devices

    def test_get_vmdk_path_and_adapter_type(self):
        filename = '[test_datastore] test_file.vmdk'
        devices = self._vmdk_path_and_adapter_type_devices(filename)
        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None)
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertEqual(filename, vmdk.path)

    def test_get_vmdk_path_and_adapter_type_with_match(self):
        n_filename = '[test_datastore] uuid/uuid.vmdk'
        devices = self._vmdk_path_and_adapter_type_devices(n_filename)
        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None, uuid='uuid')
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertEqual(n_filename, vmdk.path)

    def test_get_vmdk_path_and_adapter_type_with_nomatch(self):
        n_filename = '[test_datastore] diuu/diuu.vmdk'
        session = fake.FakeSession()
        devices = self._vmdk_path_and_adapter_type_devices(n_filename)
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None, uuid='uuid')
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertIsNone(vmdk.path)

    def test_get_vmdk_adapter_type(self):
        # Test for the adapter_type to be used in vmdk descriptor
        # Adapter type in vmdk descriptor is same for LSI-SAS, LSILogic
        # and ParaVirtual
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("lsiLogic")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("lsiLogicsas")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("paraVirtual")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("dummyAdapter")
        self.assertEqual("dummyAdapter", vmdk_adapter_type)

    def test_find_allocated_slots(self):
        disk1 = fake.VirtualDisk(200, 0)
        disk2 = fake.VirtualDisk(200, 1)
        disk3 = fake.VirtualDisk(201, 1)
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        scsi0 = fake.VirtualLsiLogicController(key=1000, scsiCtlrUnitNumber=7)
        devices = [disk1, disk2, disk3, ide0, ide1, scsi0]
        taken = vm_util._find_allocated_slots(devices)
        self.assertEqual([0, 1], sorted(taken[200]))
        self.assertEqual([1], taken[201])
        self.assertEqual([7], taken[1000])

    def test_allocate_controller_key_and_unit_number_ide_default(self):
        # Test that default IDE controllers are used when there is a free slot
        # on them
        disk1 = fake.VirtualDisk(200, 0)
        disk2 = fake.VirtualDisk(200, 1)
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        devices = [disk1, disk2, ide0, ide1]
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            None,
                                                            devices,
                                                            'ide')
        self.assertEqual(201, controller_key)
        self.assertEqual(0, unit_number)
        self.assertIsNone(controller_spec)

    def test_allocate_controller_key_and_unit_number_ide(self):
        # Test that a new controller is created when there is no free slot on
        # the default IDE controllers
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        devices = [ide0, ide1]
        for controller_key in [200, 201]:
            for unit_number in [0, 1]:
                disk = fake.VirtualDisk(controller_key, unit_number)
                devices.append(disk)
        factory = fake.FakeFactory()
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            factory,
                                                            devices,
                                                            'ide')
        self.assertEqual(-101, controller_key)
        self.assertEqual(0, unit_number)
        self.assertIsNotNone(controller_spec)

    def test_allocate_controller_key_and_unit_number_scsi(self):
        # Test that we allocate on existing SCSI controller if there is a free
        # slot on it
        devices = [fake.VirtualLsiLogicController(1000, scsiCtlrUnitNumber=7)]
        for unit_number in range(7):
            disk = fake.VirtualDisk(1000, unit_number)
            devices.append(disk)
        factory = fake.FakeFactory()
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            factory,
                                                            devices,
                                                            'lsiLogic')
        self.assertEqual(1000, controller_key)
        self.assertEqual(8, unit_number)
        self.assertIsNone(controller_spec)

    def _test_get_vnc_config_spec(self, port):

        result = vm_util.get_vnc_config_spec(fake.FakeFactory(),
                                             port)
        return result

    def test_get_vnc_config_spec(self):
        result = self._test_get_vnc_config_spec(7)
        expected = """{'extraConfig': [
                          {'value': 'true',
                           'key': 'RemoteDisplay.vnc.enabled',
                           'obj_name': 'ns0:OptionValue'},
                          {'value': 7,
                           'key': 'RemoteDisplay.vnc.port',
                           'obj_name': 'ns0:OptionValue'}],
                       'obj_name': 'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def _create_fake_vms(self):
        fake_vms = fake.FakeRetrieveResult()
        OptionValue = collections.namedtuple('OptionValue', ['key', 'value'])
        for i in range(10):
            vm = fake.ManagedObject()
            opt_val = OptionValue(key='', value=5900 + i)
            vm.set(vm_util.VNC_CONFIG_KEY, opt_val)
            fake_vms.add_object(vm)
        return fake_vms

    def test_get_vnc_port(self):
        fake_vms = self._create_fake_vms()
        self.flags(vnc_port=5900, group='vmware')
        self.flags(vnc_port_total=10000, group='vmware')
        actual = vm_util.get_vnc_port(
            fake.FakeObjectRetrievalSession(fake_vms))
        self.assertEqual(actual, 5910)

    def test_get_vnc_port_exhausted(self):
        fake_vms = self._create_fake_vms()
        self.flags(vnc_port=5900, group='vmware')
        self.flags(vnc_port_total=10, group='vmware')
        self.assertRaises(exception.ConsolePortRangeExhausted,
                          vm_util.get_vnc_port,
                          fake.FakeObjectRetrievalSession(fake_vms))

    def test_get_all_cluster_refs_by_name_none(self):
        fake_objects = fake.FakeRetrieveResult()
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['fake_cluster'])
        self.assertEqual({}, refs)

    def test_get_all_cluster_refs_by_name_exists(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.ClusterComputeResource(name='cluster'))
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['cluster'])
        self.assertEqual(1, len(refs))

    def test_get_all_cluster_refs_by_name_missing(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(partialObject(path='cluster'))
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['cluster'])
        self.assertEqual({}, refs)

    def test_propset_dict_simple(self):
        ObjectContent = collections.namedtuple('ObjectContent', ['propSet'])
        DynamicProperty = collections.namedtuple('Property', ['name', 'val'])

        object = ObjectContent(propSet=[
                    DynamicProperty(name='foo', val="bar")])
        propdict = vm_util.propset_dict(object.propSet)
        self.assertEqual("bar", propdict['foo'])

    def test_propset_dict_complex(self):
        ObjectContent = collections.namedtuple('ObjectContent', ['propSet'])
        DynamicProperty = collections.namedtuple('Property', ['name', 'val'])
        MoRef = collections.namedtuple('Val', ['value'])

        object = ObjectContent(propSet=[
                    DynamicProperty(name='foo', val="bar"),
                    DynamicProperty(name='some.thing',
                                    val=MoRef(value='else')),
                    DynamicProperty(name='another.thing', val='value')])

        propdict = vm_util.propset_dict(object.propSet)
        self.assertEqual("bar", propdict['foo'])
        self.assertTrue(hasattr(propdict['some.thing'], 'value'))
        self.assertEqual("else", propdict['some.thing'].value)
        self.assertEqual("value", propdict['another.thing'])

    def _test_detach_virtual_disk_spec(self, destroy_disk=False):
        virtual_device_config = vm_util.detach_virtual_disk_spec(
                                                     fake.FakeFactory(),
                                                     'fake_device',
                                                     destroy_disk)
        self.assertEqual('remove', virtual_device_config.operation)
        self.assertEqual('fake_device', virtual_device_config.device)
        self.assertEqual('ns0:VirtualDeviceConfigSpec',
                         virtual_device_config.obj_name)
        if destroy_disk:
            self.assertEqual('destroy', virtual_device_config.fileOperation)
        else:
            self.assertFalse(hasattr(virtual_device_config, 'fileOperation'))

    def test_detach_virtual_disk_spec(self):
        self._test_detach_virtual_disk_spec(destroy_disk=False)

    def test_detach_virtual_disk_destroy_spec(self):
        self._test_detach_virtual_disk_spec(destroy_disk=True)

    def test_get_vm_create_spec(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        extra_specs = vm_util.ExtraSpecs()
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_allocations(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_limit=7,
                                       cpu_reservation=6)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'reservation': 6,
                              'limit': 7,
                              'obj_name': 'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_limit(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_limit=7)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'limit': 7,
                              'obj_name': 'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_share(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_shares_level='high')
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'shares': {'level': 'high',
                                         'shares': 0,
                                         'obj_name':'ns0:SharesInfo'},
                              'obj_name':'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_share_custom(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_shares_level='custom',
                                       cpu_shares_share=1948)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'shares': {'level': 'custom',
                                         'shares': 1948,
                                         'obj_name':'ns0:SharesInfo'},
                              'obj_name':'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_create_vm(self):

        method_list = ['CreateVM_Task', 'get_dynamic_property']

        def fake_call_method(module, method, *args, **kwargs):
            expected_method = method_list.pop(0)
            self.assertEqual(expected_method, method)
            if (expected_method == 'CreateVM_Task'):
                return 'fake_create_vm_task'
            elif (expected_method == 'get_dynamic_property'):
                task_info = mock.Mock(state="success", result="fake_vm_ref")
                return task_info
            else:
                self.fail('Should not get here....')

        def fake_wait_for_task(self, *args):
            task_info = mock.Mock(state="success", result="fake_vm_ref")
            return task_info

        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        fake_call_mock = mock.Mock(side_effect=fake_call_method)
        fake_wait_mock = mock.Mock(side_effect=fake_wait_for_task)
        with contextlib.nested(
                mock.patch.object(session, '_wait_for_task',
                                  fake_wait_mock),
                mock.patch.object(session, '_call_method',
                                  fake_call_mock)
        ) as (wait_for_task, call_method):
            vm_ref = vm_util.create_vm(
                session,
                fake_instance,
                'fake_vm_folder',
                'fake_config_spec',
                'fake_res_pool_ref')
            self.assertEqual('fake_vm_ref', vm_ref)

            call_method.assert_called_once_with(mock.ANY, 'CreateVM_Task',
                'fake_vm_folder', config='fake_config_spec',
                pool='fake_res_pool_ref')
            wait_for_task.assert_called_once_with('fake_create_vm_task')

    @mock.patch.object(vm_util.LOG, 'warning')
    def test_create_vm_invalid_guestid(self, mock_log_warn):
        """Ensure we warn when create_vm() fails after we passed an
        unrecognised guestId
        """

        found = [False]

        def fake_log_warn(msg, values):
            if not isinstance(values, dict):
                return
            if values.get('ostype') == 'invalid_os_type':
                found[0] = True
        mock_log_warn.side_effect = fake_log_warn

        instance_values = {'id': 7, 'name': 'fake-name',
                           'uuid': uuidutils.generate_uuid(),
                           'vcpus': 2, 'memory_mb': 2048}
        instance = fake_instance.fake_instance_obj(
            context.RequestContext('fake', 'fake', is_admin=False),
            **instance_values)

        session = driver.VMwareAPISession()

        config_spec = vm_util.get_vm_create_spec(
            session.vim.client.factory,
            instance, instance.name, 'fake-datastore', [],
            vm_util.ExtraSpecs(),
            os_type='invalid_os_type')

        self.assertRaises(vexc.VMwareDriverException,
                          vm_util.create_vm, session, instance, 'folder',
                          config_spec, 'res-pool')
        self.assertTrue(found[0])

    def test_convert_vif_model(self):
        expected = "VirtualE1000"
        result = vm_util.convert_vif_model(network_model.VIF_MODEL_E1000)
        self.assertEqual(expected, result)
        expected = "VirtualE1000e"
        result = vm_util.convert_vif_model(network_model.VIF_MODEL_E1000E)
        self.assertEqual(expected, result)
        types = ["VirtualE1000", "VirtualE1000e", "VirtualPCNet32",
                 "VirtualVmxnet"]
        for type in types:
            self.assertEqual(type,
                             vm_util.convert_vif_model(type))
        self.assertRaises(exception.Invalid,
                          vm_util.convert_vif_model,
                          "InvalidVifModel")

    def test_power_on_instance_with_vm_ref(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance,
                                      vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_without_vm_ref(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(vm_util, "get_vm_ref",
                              return_value='fake-vm-ref'),
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_get_vm_ref, fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance)
            fake_get_vm_ref.assert_called_once_with(session, fake_instance)
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_with_exception(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task",
                              side_effect=exception.NovaException('fake')),
        ) as (fake_call_method, fake_wait_for_task):
            self.assertRaises(exception.NovaException,
                              vm_util.power_on_instance,
                              session, fake_instance,
                              vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_with_power_state_exception(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(
                    session, "_wait_for_task",
                    side_effect=vexc.InvalidPowerStateException),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance,
                                      vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_create_virtual_disk(self):
        session = fake.FakeSession()
        dm = session.vim.service_content.virtualDiskManager
        with contextlib.nested(
            mock.patch.object(vm_util, "get_vmdk_create_spec",
                              return_value='fake-spec'),
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_get_spec, fake_call_method, fake_wait_for_task):
            vm_util.create_virtual_disk(session, 'fake-dc-ref',
                                        'fake-adapter-type', 'fake-disk-type',
                                        'fake-path', 7)
            fake_get_spec.assert_called_once_with(
                    session.vim.client.factory, 7,
                    'fake-adapter-type',
                    'fake-disk-type')
            fake_call_method.assert_called_once_with(
                    session.vim,
                    "CreateVirtualDisk_Task",
                    dm,
                    name='fake-path',
                    datacenter='fake-dc-ref',
                    spec='fake-spec')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_copy_virtual_disk(self):
        session = fake.FakeSession()
        dm = session.vim.service_content.virtualDiskManager
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.copy_virtual_disk(session, 'fake-dc-ref',
                                      'fake-source', 'fake-dest')
            fake_call_method.assert_called_once_with(
                    session.vim,
                    "CopyVirtualDisk_Task",
                    dm,
                    sourceName='fake-source',
                    sourceDatacenter='fake-dc-ref',
                    destName='fake-dest')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def _create_fake_vm_objects(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.VirtualMachine())
        return fake_objects

    def test_get_values(self):
        objects = self._create_fake_vm_objects()
        query = vm_util.get_values_from_object_properties(
            fake.FakeObjectRetrievalSession(objects), objects)
        self.assertEqual('poweredOn', query['runtime.powerState'])
        self.assertEqual('guestToolsRunning',
                         query['summary.guest.toolsRunningStatus'])
        self.assertEqual('toolsOk', query['summary.guest.toolsStatus'])

    def test_reconfigure_vm(self):
        session = fake.FakeSession()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake_reconfigure_task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (_call_method, _wait_for_task):
            vm_util.reconfigure_vm(session, 'fake-ref', 'fake-spec')
            _call_method.assert_called_once_with(mock.ANY,
                'ReconfigVM_Task', 'fake-ref', spec='fake-spec')
            _wait_for_task.assert_called_once_with(
                'fake_reconfigure_task')

    def test_get_network_attach_config_spec_opaque(self):
        vif_info = {'network_name': 'br-int',
            'mac_address': '00:00:00:ca:fe:01',
            'network_ref': {'type': 'OpaqueNetwork',
                            'network-id': 'fake-network-id',
                            'network-type': 'opaque'},
            'iface_id': 7,
            'vif_model': 'VirtualE1000'}
        result = vm_util.get_network_attach_config_spec(
                fake.FakeFactory(), vif_info, 1)
        card = 'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo'
        expected = """{
            'extraConfig': [{'value': 7,
                             'key': 'nvp.iface-id.1',
                             'obj_name':'ns0:OptionValue'}],
            'deviceChange': [
                {'device': {
                     'macAddress':'00:00:00:ca:fe:01',
                     'addressType': 'manual',
                     'connectable': {
                         'allowGuestControl':True,
                         'startConnected': True,
                         'connected': True,
                         'obj_name':'ns0:VirtualDeviceConnectInfo'},
                     'backing': {
                         'opaqueNetworkType': 'opaque',
                         'opaqueNetworkId': 'fake-network-id',
                         'obj_name': '%(card)s'},
                     'key': -47,
                     'obj_name': 'ns0:VirtualE1000',
                     'wakeOnLanEnabled': True},
                 'operation': 'add',
                 'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}""" % {'card': card}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_network_attach_config_spec_dvs(self):
        vif_info = {'network_name': 'br100',
            'mac_address': '00:00:00:ca:fe:01',
            'network_ref': {'type': 'DistributedVirtualPortgroup',
                            'dvsw': 'fake-network-id',
                            'dvpg': 'fake-group'},
            'iface_id': 7,
            'vif_model': 'VirtualE1000'}
        result = vm_util.get_network_attach_config_spec(
                fake.FakeFactory(), vif_info, 1)
        port = 'ns0:DistributedVirtualSwitchPortConnection'
        backing = 'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo'
        expected = """{
            'extraConfig': [{'value': 7,
                             'key': 'nvp.iface-id.1',
                             'obj_name': 'ns0:OptionValue'}],
            'deviceChange': [
                {'device': {'macAddress': '00:00:00:ca:fe:01',
                            'addressType': 'manual',
                            'connectable': {
                                'allowGuestControl': True,
                                'startConnected': True,
                                'connected': True,
                                'obj_name': 'ns0:VirtualDeviceConnectInfo'},
                 'backing': {
                     'port': {
                         'portgroupKey': 'fake-group',
                         'switchUuid': 'fake-network-id',
                         'obj_name': '%(obj_name_port)s'},
                     'obj_name': '%(obj_name_backing)s'},
                     'key': -47,
                     'obj_name': 'ns0:VirtualE1000',
                     'wakeOnLanEnabled': True},
                 'operation': 'add',
                 'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}""" % {
                    'obj_name_backing': backing,
                    'obj_name_port': port}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_network_detach_config_spec(self):
        result = vm_util.get_network_detach_config_spec(
                fake.FakeFactory(), 'fake-device', 2)
        expected = """{
            'extraConfig': [{'value': 'free',
                             'key': 'nvp.iface-id.2',
                             'obj_name': 'ns0:OptionValue'}],
            'deviceChange': [{'device': 'fake-device',
                              'operation': 'remove',
                              'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    @mock.patch.object(vm_util, "get_vm_ref", return_value="fake-vm-ref")
    def test_power_off_instance_no_vm_ref(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance)
            fake_get_ref.assert_called_once_with(session, fake_instance)
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance_with_exception(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task',
                              side_effect=exception.NovaException('fake'))
        ) as (fake_call_method, fake_wait_for_task):
            self.assertRaises(exception.NovaException,
                              vm_util.power_off_instance,
                              session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance_power_state_exception(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(
                    session, '_wait_for_task',
                    side_effect=vexc.InvalidPowerStateException)
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    def test_get_vm_create_spec_updated_hw_version(self):
        extra_specs = vm_util.ExtraSpecs(hw_version='vmx-08')
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs=extra_specs)
        self.assertEqual('vmx-08', result.version)

    def test_vm_create_spec_with_profile_spec(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        datastore = ds_util.Datastore('fake-ds-ref', 'fake-ds-name')
        extra_specs = vm_util.ExtraSpecs()
        create_spec = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            datastore.name, [],
                                            extra_specs,
                                            profile_spec='fake_profile_spec')
        self.assertEqual(['fake_profile_spec'], create_spec.vmProfile)

    @mock.patch.object(pbm, 'get_profile_id_by_name')
    def test_get_storage_profile_spec(self, mock_retrieve_profile_id):
        fake_profile_id = fake.DataObject()
        fake_profile_id.uniqueId = 'fake_unique_id'
        mock_retrieve_profile_id.return_value = fake_profile_id
        profile_spec = vm_util.get_storage_profile_spec(fake.FakeSession(),
                                                        'fake_policy')
        self.assertEqual('ns0:VirtualMachineDefinedProfileSpec',
                         profile_spec.obj_name)
        self.assertEqual(fake_profile_id.uniqueId, profile_spec.profileId)

    @mock.patch.object(pbm, 'get_profile_id_by_name')
    def test_storage_spec_empty_profile(self, mock_retrieve_profile_id):
        mock_retrieve_profile_id.return_value = None
        profile_spec = vm_util.get_storage_profile_spec(fake.FakeSession(),
                                                        'fake_policy')
        self.assertIsNone(profile_spec)

    def test_get_ephemeral_name(self):
        filename = vm_util.get_ephemeral_name(0)
        self.assertEqual('ephemeral_0.vmdk', filename)

@mock.patch.object(driver.VMwareAPISession, 'vim', stubs.fake_vim_prop)
class VMwareVMUtilGetHostRefTestCase(test.NoDBTestCase):
    # N.B. Mocking on the class only mocks test_*(), but we need
    # VMwareAPISession.vim to be mocked in both setUp and tests. Not mocking in
    # setUp causes object initialisation to fail. Not mocking in tests results
    # in vim calls not using FakeVim.
    @mock.patch.object(driver.VMwareAPISession, 'vim', stubs.fake_vim_prop)
    def setUp(self):
        super(VMwareVMUtilGetHostRefTestCase, self).setUp()
        fake.reset()
        vm_util.vm_refs_cache_reset()

        self.session = driver.VMwareAPISession()

        # Create a fake VirtualMachine running on a known host
        self.host_ref = fake._db_content['HostSystem'].keys()[0]
        self.vm_ref = fake.create_vm(host_ref=self.host_ref)

    @mock.patch.object(vm_util, 'get_vm_ref')
    def test_get_host_ref_for_vm(self, mock_get_vm_ref):
        mock_get_vm_ref.return_value = self.vm_ref

        ret = vm_util.get_host_ref_for_vm(self.session, 'fake-instance')

        mock_get_vm_ref.assert_called_once_with(self.session, 'fake-instance')
        self.assertEqual(self.host_ref, ret)

    @mock.patch.object(vm_util, 'get_vm_ref')
    def test_get_host_name_for_vm(self, mock_get_vm_ref):
        mock_get_vm_ref.return_value = self.vm_ref

        host = fake._get_object(self.host_ref)

        ret = vm_util.get_host_name_for_vm(self.session, 'fake-instance')

        mock_get_vm_ref.assert_called_once_with(self.session, 'fake-instance')
        self.assertEqual(host.name, ret)


import json
import spacy
from random import randint
from pprint import pprint
with open('MLMap1.json') as f:
  cm1 = json.load(f)

with open('MLMap2.json') as f:
  cm2 = json.load(f)

with open('MLMap3.json') as f:
  cm3 = json.load(f)

nlp = spacy.load("en_core_web_sm")

def is_valid_token(token):
        ret = True
        if token.is_stop:
            ret = False
        if token.pos_ == 'SYM':
            ret = False
        return ret

def preprocess_sentence(sentence):
    return " ".join([token.lemma_ for token in nlp(sentence) if is_valid_token(token)]).lower()

def is_similar(sentence1,sentence2,threshold):
    if sentence1.similarity(sentence2) >= threshold:
        return True
    return False

def process_map_for_group_map(group_map,cm):
    concepts = dict()
    for con in cm['concepts']:
        #concepts[con['id']] = preprocess_sentence(con['text'])
        concepts[con['id']] = con['text']
    for kc in group_map.keys():
        text = list()
        prop = list()
        key_concept = nlp(kc)
        for c in concepts.values():
            c = nlp(preprocess_sentence(c))
            if is_similar(key_concept,c,0.89):
                for pro in cm['propositions']:
                    pro_txt = nlp(preprocess_sentence(concepts[pro['from']]))
                    if is_similar(pro_txt,c,0.89):
                        prop.append(pro['text'])
                        text.append(concepts[pro['to']])
        group_map[kc]["propositions"] = group_map[kc]["propositions"] + prop
        group_map[kc]["text"] = group_map[kc]["text"] + text
    return group_map

def clean_group_map(group_map):
    for k,v in group_map.items():
        i = 0
        while i<len(v['text']):
            j = i + 1
            while j<len(v['text']):
                if is_similar(nlp(v['text'][i]),nlp(v['text'][j]),0.7):
                    del v['text'][j]
                    del v['propositions'][j]
                j+=1
            i+=1
        group_map[k] = v
    return group_map

kc_pair = dict()
key_concepts = ["machine learning", "unsupervised learning", "supervised learning",
                "applications", "deep learning" ,"reinforcement learning"]
for i,c in enumerate(key_concepts):
    key_concepts[i] = preprocess_sentence(c)

group_map = dict()
for kc in key_concepts:
    group_map[kc] = {"propositions": list(), "text": list()}
cms = [cm1,cm2,cm3]
for cm in cms:
    group_map = process_map_for_group_map(group_map,cm)
group_map = clean_group_map(group_map)

def map_per_key_concept(group_map,key_concepts):
    maps = list()
    for key_concept in key_concepts:
        x = 35
        y = 366
        json_cm = dict()
        json_cm['concepts'] = []
        json_cm['propositions'] = []
        id_gen = 0
        concepts_dict = dict()
        concepts_dict[key_concept] = id_gen
        con_entry = dict()
        con_entry['text'] = key_concept
        con_entry['id'] = id_gen
        con_entry["x"] = 762
        con_entry["y"] = 111
        json_cm['concepts'].append(con_entry)
        id_gen += 1
        for concept in group_map[key_concept]['text']:
            concepts_dict[concept] = id_gen
            con_entry = dict()
            con_entry['text'] = concept
            con_entry['id'] = id_gen
            con_entry["x"] = x + randint(30,50)
            x += 230
            con_entry["y"] = y
            json_cm['concepts'].append(con_entry)
            id_gen += 1
        for pro,txt in enumerate(group_map[key_concept]['text']):
            pro_entry = dict()
            pro_entry['from'] = 0
            pro_entry['to'] = concepts_dict[txt]
            pro_entry['text'] = group_map[key_concept]['propositions'][pro]
            json_cm['propositions'].append(pro_entry)
        maps.append(json_cm)
    for i,mapp in enumerate(maps):
        maps[i] = json.dumps(mapp)
    return maps

maps = map_per_key_concept(group_map,key_concepts)
print(maps[0])


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 26 11:27:31 2022

@author: matte
"""
import random
import numpy as np
import torch

from models import BiSeNetV2
from copy import deepcopy
from collections import OrderedDict
from torch.utils.data import DataLoader
from tqdm import tqdm
from sklearn.metrics import confusion_matrix
DEVICE = "cuda"
NUM_CLASSES = 19
NUM_ROUNDS = 1600
NUM_CLIENTS_FOR_ROUND = 5

T = 5
def _fast_hist(n_classes, label_true, label_pred):
      mask = (label_true >= 0) & (label_true < n_classes)
      hist = np.bincount(
          n_classes * label_true[mask].astype(int) + label_pred[mask],
          minlength=n_classes ** 2,
      ).reshape(n_classes, n_classes)
      return hist

def compute_mIoU(y_true,y_pred):

  y_pred = y_pred.cpu().detach().numpy().flatten()
  y_true = y_true.cpu().detach().numpy().flatten()
  # print(y_true)
  # print(y_pred)
  # index = [i for i in range(len(y_true)) if y_true[i] != 255]
  # mask = y_true == 255
  # y_true = np.delete(y_true, np.where(mask))
  # y_pred = np.delete(y_pred, np.where(mask))
  #print(index)
  # print(y_true)
  # print(y_pred)
  #hist = confusion_matrix(y_true, y_pred, labels=range(19))
  hist = _fast_hist(19,y_true,y_pred)
  #print(hist)
  gt_sum = hist.sum(axis=1)
  mask = (gt_sum != 0)
  diag = np.diag(hist)
  iu = diag / (gt_sum + hist.sum(axis=0) - diag)
  mean_iu = np.mean(iu[mask])
  return mean_iu

# def compute_mIoU(mask,pred_mask,smooth=1e-10,n_classes=19):
#     with torch.no_grad():
#         #pred_mask = F.softmax(pred_mask, dim=1)
#         pred_mask = torch.argmax(pred_mask, dim=1)
#         pred_mask = pred_mask.contiguous().view(-1)
#         mask = mask.contiguous().view(-1)

#         iou_per_class = []
#         for clas in range(0, n_classes): #loop per pixel class
#             true_class = pred_mask == clas
#             true_label = mask == clas

#             if true_label.long().sum().item() == 0: #no exist label in this loop
#                 iou_per_class.append(np.nan)
#             else:
#                 intersect = torch.logical_and(true_class, true_label).sum().float().item()
#                 union = torch.logical_or(true_class, true_label).sum().float().item()

#                 iou = (intersect + smooth) / (union +smooth)
#                 iou_per_class.append(iou)
#         return np.nanmean(iou_per_class)

# def compute_mIoU(y_true, y_pred,n_classes):
#      # ytrue, ypred is a flatten vector
#      y_pred = y_pred.cpu().flatten()
#      y_true = y_true.cpu().flatten()
#      labels = range(19)
#      current = confusion_matrix(y_true, y_pred, labels=labels)
#      # compute mean iou
#      intersection = np.diag(current)
#      ground_truth_set = current.sum(axis=1)
#      predicted_set = current.sum(axis=0)
#      union = ground_truth_set + predicted_set - intersection
#      IoU = intersection / union.astype(np.float32)
#      return np.mean(IoU)

class Server():

    def __init__(self,clients,path,test_dataloader: DataLoader,pseudo_lab=False):
        self.test_dataloader=test_dataloader
        self.clients = clients
        self.main_model = BiSeNetV2(n_classes=19,output_aux=True,pretrained=True)
        if pseudo_lab:
          self.main_model.load_state_dict(path,strict=False)

        self.pseudo_lab=pseudo_lab

    def load_server_model_on_client(self,client):
      client.bisenet_model.load_state_dict(deepcopy(self.main_model.state_dict()))

    def _server_opt(self, pseudo_gradient):

        for n, p in self.model.named_parameters():
            p.grad = -1.0 * pseudo_gradient[n]

        self.optimizer.step()

        bn_layers = OrderedDict(
            {k: v for k, v in pseudo_gradient.items() if "running" in k or "num_batches_tracked" in k})
        self.model.load_state_dict(bn_layers, strict=False)

    def _aggregation(self):
        total_weight = 0.
        base = OrderedDict()

        for (client_samples, client_model) in self.updates:

            total_weight += client_samples
            for key, value in client_model.items():
                if key in base:
                    base[key] += client_samples * value.type(torch.FloatTensor)
                else:
                    base[key] = client_samples * value.type(torch.FloatTensor)
        averaged_sol_n = deepcopy(self.model_params_dict)
        for key, value in base.items():
            if total_weight != 0:
                averaged_sol_n[key] = value.to('cuda') / total_weight

        return averaged_sol_n

    def _get_model_total_grad(self):
        total_norm = 0
        for name, p in self.model.named_parameters():
            if p.requires_grad:
                param_norm = p.grad.data.norm(2)
                total_norm += param_norm.item() ** 2
        total_grad = total_norm ** 0.5
        self.writer.write(f"total grad norm: {round(total_grad, 2)}")  # 0: no gradients server side
        return total_grad

    def update_model(self):
        """FedAvg on the clients' updates for the current round.
        Weighted average of self.updates, where the weight is given by the number
        of samples seen by the corresponding client at training time.
        Saves the new central model in self.client_model and its state dictionary in self.model
        """

        averaged_sol_n = self._aggregation()

        if self.optimizer is not None:  # optimizer step
            self._server_opt(averaged_sol_n)
            self.total_grad = self._get_model_total_grad()
        else:
            self.model.load_state_dict(averaged_sol_n, strict=False)
        self.model_params_dict = deepcopy(self.model.state_dict())

        self.updates = []

    def evaluate(self):
      net = self.main_model.to(DEVICE) # this will bring the network to GPU if DEVICE is cuda
      net.train(False) # Set Network to evaluation mode
      net=net.half()
      running_corrects = 0
      torch.cuda.empty_cache()
      mIoU = 0
      count = 0
      for images, labels in self.test_dataloader:
        images = images.half().to(DEVICE)
        labels = labels.half().to(DEVICE)
        #print("images:"+str(images.size()))
        #print("labels:"+str(labels.size()))
        # Forward Pass
        outputs = net(images,test=True,use_test_resize=False)
        preds = outputs.argmax(dim=1)
        # print(outputs.size())
        mIoU += compute_mIoU(labels,preds)
        #print(mIoU)
        count += 1

        # iou = ops.box_iou(labels, outputs)

        # print('IOU : ', iou.numpy()[0][0])
      print("\nmIoU = ",mIoU/count)
      net.train(True)
      return mIoU/count

    def train(self):
        check_round = 0
        mIoU_for_rounds = []
        loss_train = []

        # checkpoint = torch.load('/content/drive/MyDrive/step5/FDA/DatasetA_I/checkpoints/T1/450checkpoint.pt')
        # self.main_model.load_state_dict(checkpoint['model_state_dict'])
        # self.main_model.to(DEVICE)
        # check_round = checkpoint['round']
        # mIoU_for_rounds = checkpoint['mIoU_for_rounds']
        # loss_train = checkpoint['loss_train']

        if self.pseudo_lab:
          for client in self.clients:
            client.criterion.set_teacher(deepcopy(self.main_model).half())
        for i in range(check_round,NUM_ROUNDS):

            new_weights = []
            num_samples_list = []
            tot_samples = 0

            for client in random.sample(self.clients,NUM_CLIENTS_FOR_ROUND):
              if self.pseudo_lab:
                if T!=0 and i % T == 0:
                  print("update teacher model...")
                  client.criterion.set_teacher(deepcopy(self.main_model).half())

              self.load_server_model_on_client(client)
              num_samples, update,loss = client.train()
              #new_weights.append((num_samples, update))

              new_weights.append(update)
              tot_samples += num_samples
              num_samples_list.append(num_samples)

            print(f"[Server]: round {i}")

            mean_weights = OrderedDict()
            keys = list(new_weights[0].keys())

            # Loop through the keys
            for key in keys:
                # Initialize a list to store the values corresponding to the key
                values = []

                # Loop through the OrderedDict objects
                for n_s,d in zip(num_samples_list,new_weights):
                    # Add the value corresponding to the key to the list
                    values.append(d[key]*n_s)

                # Calculate the mean value
                mean_value = sum(values) / tot_samples

                # Add the key and the mean value to the mean_dict
                mean_weights[key] = mean_value
            self.main_model.load_state_dict(mean_weights,strict=False)

            if i % 5 == 0:
              mIoU_for_rounds.append(self.evaluate())
              loss_train.append(loss)
            if i % 10 == 0:
              torch.save({
              'round': i,
              'model_state_dict': self.main_model.state_dict(),
              'mIoU_for_rounds' : mIoU_for_rounds,
              'loss_train':loss_train}, '/content/drive/MyDrive/step5/FDA/DatasetA_I/checkpoints/T5/'+ str(i) +'checkpoint.pt')

        return self.main_model


#Input - Ross, Rachel, Monica, Chandler, Joey, Phoebe

import random
# Split string method
names_string = input("Give me everybody's names, separated by a comma. ")
names = names_string.split(", ")
#print(names_string)
#print(names)
n=len(names)
num=random.randint(0,n)

print(f"{names[num]} is going to buy the meal today!")