"""
Utility functions for data loading, downloading, display
"""
__author__ = "Marcel Kurovski"
__copyright__ = "Marcel Kurovski"
__license__ = "mit"

import json
import logging
import os
import requests
import shutil
import sys
import time
import zipfile

import matplotlib.pyplot as plt

from graphpipe import remote
import numpy as np
from scipy.io import loadmat

EMNIST_MATLAB_URL = 'http://www.itl.nist.gov/iaui/vip/cs_links/EMNIST/matlab.zip'
EMNIST_FILENAME = 'emnist-byclass.mat'

_logger = logging.getLogger(__name__)

def setup_logging(loglevel):
    """Setup basic logging

    Args:
      loglevel (int): minimum loglevel for emitting messages
    """
    logformat = "[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
    logging.basicConfig(level=loglevel, stream=sys.stdout,
                        format=logformat, datefmt="%Y-%m-%d %H:%M:%S")

setup_logging(logging.INFO)

def load_emnist(emnist_folder_path, download=False):
    """
    Loads EMNIST data from folder (eventually downloads first)
    using the Matlab format
    See https://www.nist.gov/itl/iad/image-group/emnist-dataset

    Args:
        emnist_folder_path (str): folder that contains EMNIST Matlab files
        download (bool): in case `emnist_folder_path` does not exist or contain
                         the Matlab files, download the data and extract them

    Returns:
        x_train (:obj:`np.array`): (n_train, 28, 28) training images
        y_train (:obj:`np.array`): (n_train, 1) training labels
        x_test (:obj:`np.array`): (n_test, 28, 28) training images
        y_test (:obj:`np.array`y): (n_test, 1) training labels
        mapping (:obj:`np.array`): (62, 2) maps training labels (first column)
            to ascii codes (second column)
    """
    filepath = os.path.join(emnist_folder_path, EMNIST_FILENAME)

    if not (os.path.isdir(emnist_folder_path) and os.path.isfile(filepath)):
        if download:
            download_emnist(emnist_folder_path)
        else:
            error_msg = ("Folder {} or file {} does not exist "
                         "and download is deactivated").format(
                                emnist_folder_path, EMNIST_FILENAME)
            raise FileNotFoundError(error_msg)

    raw_emnist_data = loadmat(filepath)

    _logger.info("Loading train and test data from %s", filepath)

    # load train data
    x_train = raw_emnist_data["dataset"][0][0][0][0][0][0].astype(np.float32)
    x_train = x_train.reshape((x_train.shape[0], 28, 28), order='A')
    y_train = raw_emnist_data["dataset"][0][0][0][0][0][1]

    # load test data
    x_test = raw_emnist_data["dataset"][0][0][1][0][0][0].astype(np.float32)
    x_test = x_test.reshape((x_test.shape[0], 28, 28), order='A')
    y_test = raw_emnist_data["dataset"][0][0][1][0][0][1]

    # load mapping from label to chr
    mapping = raw_emnist_data["dataset"][0][0][2]

    return x_train, y_train, x_test, y_test, mapping

def download_emnist(emnist_folder_path):
    """
    Downloads, extracts and moves files to desired path

    Args:
        emnist_folder_path (str): folder for EMNIST data download and extraction
    """
    _logger.info("Data not found. Starting EMNIST Download from %s",
                 EMNIST_MATLAB_URL)
    if not os.path.isdir(emnist_folder_path):
        os.mkdir(emnist_folder_path)
        _logger.info("Target folder not found. Created %s", emnist_folder_path)

    emnist_matlab_zip_filepath = os.path.join(emnist_folder_path,
                                              'emnist_matlab.zip')
    req = requests.get(EMNIST_MATLAB_URL, stream=True)

    with open(emnist_matlab_zip_filepath, 'wb') as fp:
        for chunk in req.iter_content(chunk_size=1024):
            if chunk:
                fp.write(chunk)

    _logger.info("Download successful. Extracting and moving files...")
    with zipfile.ZipFile(emnist_matlab_zip_filepath, 'r') as zip_ref:
        zip_ref.extractall(emnist_folder_path)

    matlab_folder_path = os.path.join(emnist_folder_path, 'matlab')
    for matlab_file in os.listdir(matlab_folder_path):
        shutil.move(os.path.join(matlab_folder_path, matlab_file),
                    os.path.join(emnist_folder_path, matlab_file))

    shutil.rmtree(matlab_folder_path)

def get_emnist_mapping():
    """
    Create Mapping for 62 classes from ASCII code to digit or character

    Returns:
        mapping (dict): maps ASCII integer to digit / character string
    """
    ascii_digits = list(range(48, 58))
    ascii_uppercase_letters = list(range(65, 65+26))
    ascii_lowercase_letters = list(range(97, 97+26))

    digits = [chr(value) for value in ascii_digits]
    uppercase_letters = [chr(value) for value in ascii_uppercase_letters]
    lowercase_letters = [chr(value) for value in ascii_lowercase_letters]

    mapping = dict(zip(range(10+2*26),
                       digits+uppercase_letters+lowercase_letters))

    return mapping

def show_img(idx, x_train, y_train, x_test, y_test, mapping, mode='train'):
    """
    Shows the selected EMNIST image and its label

    Args:
        idx(int):
        x_train (:obj:`np.array`): (n_train, 28, 28) training images
        y_train (:obj:`np.array`): (n_train, 1) training labels
        x_test (:obj:`np.array`): (n_test, 28, 28) training images
        y_test (:obj:`np.array`y): (n_test, 1) training labels
        mapping (:obj:`np.array`): (62, 2) maps training labels (first column)
            to ascii codes (second column)
        mode (str): select image either from `train` or `test` dataset
    """
    if mode == 'train':
        img = x_train[idx]
        label = mapping[y_train[idx, 0]]
    else:
        img = x_test[idx]
        label = mapping[y_test[idx, 0]]
    plt.figure()
    plt.gray()
    plt.imshow(img)
    plt.show()
    print("Label: {}".format(label))

def show_train_progress(iteration, train_loss, test_loss, train_acc, test_acc):
    print(("Iteration: {:06d}, "
           "Loss (Train/Test): {:.3f} / {:.3f}, "
           "Accuracy: {:.3%} / {:.3%}").format(iteration,
                                               train_loss, test_loss,
                                               train_acc, test_acc))

def eval_serving_performance(n_examples, n_print_examples, request_url, seed=42,
                             dataset='test', use_graphpipe=False,
                             emnist_folder_path='emnist_data/'):
    """
    Fires queries against a model service, evaluates the result in terms
    of classification accuracy and processing time, and prints examples

    Args:
        n_examples (int): number of queries
        n_print_examples (int): number of examples to print img and classification
        request_url (str): URL of model service used for classification
        seed (int): seed for NumPy random number generator
        dataset (str): `train` or `test` data to pick evaluation examples from
        use_graphpipe (bool): use graphpipe client to execute queries (uses
                                flatbuffers instead)
        emnist_folder_path (str): folder containing EMNIST data

    Returns:
        durations ([int]): list of ms every request took
    """
    x_train, y_train, x_test, y_test, _ = load_emnist(emnist_folder_path)
    mapping = get_emnist_mapping()

    acc = 0
    durations = []

    if dataset == 'train':
        x_eval, y_eval = x_train, y_train
    else:
        x_eval, y_eval = x_test, y_test

    np.random.seed(seed)
    eval_img_indices = np.random.choice(np.arange(x_eval.shape[0]),
                                        n_examples,
                                        replace=False)

    for idx, test_img_idx in enumerate(eval_img_indices):
        test_img_flatten = x_eval[test_img_idx].reshape(1, 784) / 255

        if not use_graphpipe:
            start = time.time()
            test_img_payload = {'instances': test_img_flatten.tolist()}
            test_img_payload = json.dumps(test_img_payload)
            test_img_softmax_pred = requests.post(request_url,
                                                  data=test_img_payload)
            test_img_softmax_pred = test_img_softmax_pred.json()['predictions']
        else:
            start = time.time()
            test_img_softmax_pred = remote.execute(request_url, test_img_flatten)

        durations.append(int((time.time() - start) * 1000000)/1000)
        test_img_class_pred = np.argmax(test_img_softmax_pred)
        acc += (test_img_class_pred == y_eval[test_img_idx][0])

        # print the first 10 images with their true and predicted label
        if idx < n_print_examples:
            show_img(test_img_idx, x_train, y_train, x_test, y_test, mapping,
                     mode=dataset)
            print("Predicted Label: {}".format(mapping[test_img_class_pred]))

    print("Accuracy on {} test images: {:.2%}".format(n_examples,
                                                      acc / n_examples))

    return durations

def eval_throughput(duration, request_url, batch_size=1, dataset='test',
                    use_graphpipe=False, emnist_folder_path='emnist_data/'):
    """
    Performs a Throughput test running sequential queries against webservice
    counting the number of requests performed within duration

    Args:
        duration (int): number of seconds for running test queries
        request_url (str): URL of model service used for classification
        batch_size (int): no. of examples per batch
        dataset (str): `train` or `test` data to pick evaluation examples from
        use_graphpipe (bool): use graphpipe client to execute queries (uses
                                flatbuffers instead)
        emnist_folder_path (str): folder containing EMNIST data

    Returns:
        num_reqs (int): total number of requests performed
        reqs_per_second (float): thoughput as the number of requests performed
                                 per second

    """
    x_train, _, x_test, _, _ = load_emnist(emnist_folder_path)
    if dataset == 'train':
        data = x_train
    else:
        data = x_test
    data = data.reshape(-1, 1, 28 * 28) / 255

    num_reqs = 0
    _logger.info("Throughput Evaluation on URL {} for {} seconds ...".format(
            request_url, duration
    ))

    if not use_graphpipe:
        end_time = time.time() + duration
        while time.time() <= end_time:
            batch = data[num_reqs:(num_reqs+batch_size)].reshape(-1, 28*28)
            test_img_payload = \
                {"instances": batch.tolist()}
            test_img_payload = json.dumps(test_img_payload)
            test_img_softmax_pred = requests.post(request_url,
                                                  data=test_img_payload)
            test_img_softmax_pred.json()['predictions']
            num_reqs += batch_size
    else:
        end_time = time.time() + duration
        while time.time() <= end_time:
            remote.execute(request_url, data[num_reqs:(num_reqs+batch_size)])
            num_reqs += batch_size

    reqs_per_second = num_reqs / duration

    _logger.info(("Throughput Summary:\nURL: {}\nDuration: {} [s]\n"
                  + "Total Requests: {}\nRequests/Second: {:.2f}").format(
            request_url, duration, num_reqs, reqs_per_second
    ))

    return num_reqs, reqs_per_second


import numpy as np

def gauss_jordan(A, b):
    '''
    A is (n, n) non-singular matrix
    Equation Ax = b
    Parameters:
    ------------
    A - square (n, n) size matrix
    b - n size column
    x - n size column
    '''
    n = b.shape[0]
    block = np.c_[A, b].astype(np.float64)
    for i in range(n - 1):

        # first element need to be nonzero
        p = np.argmax(np.abs(block[i:, i])) + i
        if block[p, i] == 0:
            print('No unique solution')
            return

        if p != i:
            block[[i, p]] = block[[p, i]]

        # elimination process
        block[i+1:] -= np.outer(block[i+1:, i], block[i])/block[i, i]
    if block[-1, -2] == 0:
        print('No unique solution')
        return

    # diagonalizing matrix
    for i in range(n-2, -1, -1):
        coefs = np.array([block[i, k]/block[k, k] for k in range(i+1, n)])
        block[i, -1] -= np.dot(coefs, block[i+1:, -1])

    block[:, :-1] = np.diag(np.diag(block[:, :-1]))
    x = block[:, -1]/np.diag(block[:, :-1])

    return x


import tkinter as tk
from tkinter import filedialog
from tkinter import ttk
from ttkthemes import ThemedTk
from tkinter import messagebox
import csv
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats
from scipy.stats import norm
import wx
import wx.grid
import pandas as pd
import plotly.express as px
import re

# Use the Agg backend for non-interactive environments
matplotlib.use('TkAgg')

# Initialize a dictionary to count digits from 1 to 9
digit_counts = {str(i): 0 for i in range(1, 10)}

# Initialize a variable to count the total number of digits
total_digit_count = 0

# Create the main application window
root = ThemedTk(theme="radiance")
root.title("CSV File Processor")

# Initialize Tkinter variables for selected column names as globals
var_col1 = tk.StringVar()
var_col2 = tk.StringVar()
var_col3 = tk.StringVar()

# Create a variable to track checkbox state
checkbox_var = tk.BooleanVar()

# Function to handle the file selection
def browse_file():
    global var_col1, var_col2, var_col3  # Declare them as global
    file_path = filedialog.askopenfilename(filetypes=[
        ("CSV Files", "*.csv"),
        ("Advanced Record Definition Editor Files", "*.are"),
        ("AS400 Files", "*.as400"),
        ("dBASE Files", "*.dbf"),
        ("Microsoft Access Files", "*.mdb *.accdb"),
        ("Microsoft Excel Files", "*.xls *.xlsx"),
        ("ODBC Files", "*.odbc"),
        ("Print Report Files", "*.pr"),
        ("Adobe PDF Files", "*.pdf"),
        ("SAP/AIS Files", "*.sap"),
        ("Text Files", "*.txt"),
        ("XML Files", "*.xml"),
    ])

    if file_path:
        entry_file_path.delete(0, tk.END)  # Clear the entry field
        entry_file_path.insert(0, file_path)  # Set the selected file path

        # Get the list of columns based on the file type
        if file_path.endswith((".csv", ".txt", ".xml")):
            with open(file_path, newline='') as csvfile:
                csvreader = csv.reader(csvfile)
                header = next(csvreader)  # Read the header row
                column_names = header  # Get column names from the header row

        # Call a function to handle the retrieved column names
        handle_column_names(column_names)

    if file_path:
        # Load the CSV file
        df = pd.read_csv(file_path)

    if file_path:
        # Load the CSV file
        df = pd.read_csv(file_path)

        # Initialize variables for largest, smallest, and negative values
        largest_value = None
        smallest_value = None

        for col in df.columns:
            for value in df[col]:
                if isinstance(value, str) and re.match(r'\$\d+\.*\d*', value):
                    numeric_value = float(re.sub('[^\d.]', '', value))
                    if largest_value is None or numeric_value > largest_value:
                        largest_value = numeric_value
                    if smallest_value is None or numeric_value < smallest_value:
                        smallest_value = numeric_value

        # Get the number of records
        num_records = len(df)

        # Create a modal window to display the information
        modal = tk.Toplevel(root)
        modal.title("File Information")

        # Display the information
        label1 = tk.Label(modal, text=f"Number of Records: {num_records}")
        label2 = tk.Label(modal, text=f"Largest Value: {largest_value}")
        label3 = tk.Label(modal, text=f"Smallest Value: {smallest_value}")

        label1.pack()
        label2.pack()
        label3.pack()

# Function to create dropdown menus for column names
def handle_column_names(column_names):
    global var_col1, var_col2, var_col3  # Access the global variables

    # Create dropdown menus for each column
    dropdown_menu1 = ttk.OptionMenu(root, var_col1, *column_names)
    dropdown_menu2 = ttk.OptionMenu(root, var_col2, *column_names)
    dropdown_menu3 = ttk.OptionMenu(root, var_col3, *column_names)

    # Label each dropdown menu
    label_menu1 = ttk.Label(root, text="Account Description:")
    label_menu2 = ttk.Label(root, text="Transaction Description:")
    label_menu3 = ttk.Label(root, text="Transactions:")
    label_column_instructions1 = ttk.Label(root, text="Select 3 columns to run analysis on. The first column is the category you want the data separated by.")
    label_column_instructions2 = ttk.Label(root, text="The second is who/what bought the item/service and the last should be the transactions")

    # Grid layout for labels and dropdown menus
    label_column_instructions1.grid(row=7, column=0, columnspan=2, pady=1)
    label_column_instructions2.grid(row=8, column=0, columnspan=2, pady=1)
    label_menu1.grid(row=9, column=0, padx=1, pady=5)
    dropdown_menu1.grid(row=9, column=1, padx=1, pady=5)
    label_menu2.grid(row=10, column=0, padx=1, pady=5)
    dropdown_menu2.grid(row=10, column=1, padx=1, pady=5)
    label_menu3.grid(row=11, column=0, padx=1, pady=5)
    dropdown_menu3.grid(row=11, column=1, padx=1, pady=5)

# Define a function to clean and convert 'Total' values to float
def clean_and_convert_total(total_str_or_float):
    if isinstance(total_str_or_float, str):
        total_str = total_str_or_float.replace('$', '').replace(',', '').strip()
        # Handle negative numbers enclosed in parentheses
        if total_str.startswith('(') and total_str.endswith(')'):
            total_str = '-' + total_str[1:-1]
        try:
            return float(total_str)
        except ValueError:
            return np.nan
    elif isinstance(total_str_or_float, float):
        # If it's already a float, return it as is
        return total_str_or_float

# Function to find outliers
def process_gaussian_distribution(csv_file_path, col1, col3, checkbox_var):
    try:
        # Load transaction data from the selected CSV file
        data = pd.read_csv(csv_file_path)
        col1 = var_col1.get()
        col3 = var_col3.get()
        check = checkbox_var

        # Clean and convert the entire col3 column to float
        data[col3] = data[col3].apply(clean_and_convert_total)

        # Group transactions by 'Item'
        grouped_data = data.groupby([col1])

        # Plot Gaussian distributions for each selected column
        for i, group_data in grouped_data:

            if check:
                group_data = group_data[group_data[col3] >= 0]  # Exclude negative values

            # Fit a normal distribution to the data for the current column
            mu, std = norm.fit(group_data[col3])

            # Create a histogram
            plt.figure(figsize=(10, 6))
            plt.hist(group_data[col3], bins=30, density=True, alpha=0.6, color='blue')

            # Create a range of values for the x-axis based on the normal distribution
            xmin, xmax = plt.xlim()
            x = np.linspace(xmin, xmax, 100)
            p = norm.pdf(x, mu, std)

            # Plot the fitted normal distribution
            plt.plot(x, p, 'k', linewidth=2)
            title = f"Fit results for {i}: mu = {mu:.2f}, std = {std:.2f}"
            plt.title(title)

            plt.xlabel('Dollars Spent (Total)')
            plt.ylabel('')

            # Calculate and plot the first standard deviation lines
            std1_left = mu - std
            std1_right = mu + std
            plt.axvline(x=std1_left, color='k', linestyle='--', label='1st std dev', linewidth=2)
            plt.axvline(x=std1_right, color='k', linestyle='--', linewidth=2)

            # Calculate and plot the second standard deviation lines
            std2_left = mu - 2 * std
            std2_right = mu + 2 * std
            plt.axvline(x=std2_left, color='r', linestyle='--', label='2nd std dev', linewidth=2)
            plt.axvline(x=std2_right, color='r', linestyle='--', linewidth=2)

            # Force y-axis ticks to display whole numbers
            yint = range(min([int(val) for val in plt.yticks()[0]]), max([int(val) for val in plt.yticks()[0]]) + 1)
            plt.yticks(yint)
            plt.show()
    except ValueError as e:
        print(f"Error processing data for item {i}: {e}")

    except FileNotFoundError:
        print("CSV file not found. Please select a valid CSV file.")

# Function to process the selected CSV file and run Benford's Law analysis
def process_benfords_law():
    global total_digit_count  # Declare total_digit_count as a global variable
    global digit_counts  # Declare digit_counts as a global variable

    try:
        csv_file_path = entry_file_path.get()
        if csv_file_path:
            # Clear the previous counts
            digit_counts = {str(i): 0 for i in range(1, 10)}
            total_digit_count = 0

            # Open the CSV file and read its contents
            with open(csv_file_path, newline='') as csvfile:
                csvreader = csv.reader(csvfile)
                for row in csvreader:
                    # Join the elements in each row to form a single string
                    row_str = ''.join(row)

                    # Use regular expressions to find all leading digits in the row
                    leading_digits = re.findall(r'\b[1-9]\d*', row_str)

                    # Iterate through the leading digits
                    for digit_str in leading_digits:
                        digit = digit_str[0]  # Get the first character (leading digit)
                        digit_counts[digit] += 1
                        total_digit_count += 1

            # Calculate percentages for each digit
            digit_percentages = {digit: (count / total_digit_count) * 100 if total_digit_count > 0 else 0 for digit, count in digit_counts.items()}

            # Calculate expected Benford's Law percentages
            expected_percentages = {str(digit): np.log10(1 + 1 / digit) * 100 for digit in range(1, 10)}

            # Plot the bar chart
            digits = list(digit_percentages.keys())
            percentages = list(digit_percentages.values())
            expected_percentages_values = list(expected_percentages.values())

            # Determine bar colors based on alignment with Benford's Law
            bar_colors = ['blue' if abs(percentage - expected_percentages[digit]) <= 5 else
                        'yellow' if abs(percentage - expected_percentages[digit]) <= 7.5 else
                        'orange' if abs(percentage - expected_percentages[digit]) <= 10 else
                        'red' for digit, percentage in digit_percentages.items()]

            # Plot the bars with colors
            plt.bar(digits, percentages, color=bar_colors, label='Actual Percentages')

            # Plot the expected Benford's Law line (red)
            plt.plot(digits, expected_percentages_values, 'r', label='Benford\'s Law')

            plt.xlabel('Leading Digit')
            plt.ylabel('Percentage (%)')
            plt.title('Leading Digit Percentages vs. Benford\'s Law')
            plt.legend()
            plt.show()

    except FileNotFoundError:
        print("CSV file not found. Please select a valid CSV file.")

# Function to run transaction analysis
def analyze_transactions_from_csv(csv_file_path, col1, col2, col3):
    try:
        # Load transaction data from the selected CSV file
        data = pd.read_csv(csv_file_path)
        col1 = var_col1.get()
        col2 = var_col2.get()
        col3 = var_col3.get()

        # Clean and convert 'Total' column values
        data[col3] = data[col3].apply(clean_and_convert_total)

        # Create an interactive horizontal histogram using Plotly
        fig = px.bar(data, x=data[col3], y=data[col2], color=data[col1], orientation='h',
                     title='Total Spending by Rep',
                     labels={'Total': 'Dollars Spent (Total)', 'Rep': 'Rep'})

        # Show the interactive plot
        fig.show()

    except FileNotFoundError:
        print("CSV file not found. Please select a valid CSV file.")

# Class that creates the Duplicates modal to display duplicates found in the csv
class DuplicateViewer(wx.Frame):
    def __init__(self, duplicate_rows):
        super().__init__(None, title="Duplicate Rows Viewer", size=(800, 600))
        self.panel = wx.Panel(self)

        # Create a grid
        self.grid = wx.grid.Grid(self.panel)
        self.grid.CreateGrid(len(duplicate_rows) + 1, duplicate_rows.shape[1])  # Create a grid with one extra row for headers

        # Set column labels from DataFrame
        for col, col_label in enumerate(duplicate_rows.columns):
            self.grid.SetCellValue(0, col, col_label)

        # Populate the grid with data
        for row, (_, row_data) in enumerate(duplicate_rows.iterrows()):
            for col, cell_value in enumerate(row_data):
                self.grid.SetCellValue(row + 1, col, str(cell_value))  # Use str() to convert values to strings

        # Auto-size columns
        for col in range(duplicate_rows.shape[1]):
            self.grid.AutoSizeColumn(col)

        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.grid, 1, wx.EXPAND)
        self.panel.SetSizer(sizer)

        self.Show()

# Function to find duplicates in the data
def find_duplicate_rows(csv_file_path, col1, col2, col3):
    try:
        # Load transaction data from the selected CSV file
        data = pd.read_csv(csv_file_path)
        col1 = var_col1.get()
        col2 = var_col2.get()
        col3 = var_col3.get()

        # Identify duplicate rows based on 'Item', 'Rep', and 'Total' columns
        duplicate_rows = data[data.duplicated(subset=[col1, col2, col3], keep=False)]

        if not duplicate_rows.empty:
            # Create a wxPython application
            app = wx.App()
            DuplicateViewer(duplicate_rows)
            app.MainLoop()

            # Save the duplicate rows to a new CSV file
            duplicate_rows.to_csv('duplicate_rows.csv', index=False)

            print(f"Duplicate rows saved to 'duplicate_rows.csv'.")
        else:
            print("No duplicate rows found in the CSV file.")

    except FileNotFoundError:
        print("CSV file not found. Please select a valid CSV file.")

# Class that creates the file view modal to display the csv file in an excel-like format
class DataViewer(wx.Frame):
    def __init__(self, data):
        super(DataViewer, self).__init__(None, title="Data Viewer")
        self.panel = wx.Panel(self)

        # Create a grid to display the data
        self.grid = wx.grid.Grid(self.panel)
        self.grid.CreateGrid(data.shape[0], data.shape[1])

        # Set column labels
        for col, col_label in enumerate(data.columns):
            self.grid.SetColLabelValue(col, col_label)

        # Populate the grid with data
        for row, row_data in data.iterrows():
            for col, value in enumerate(row_data):
                self.grid.SetCellValue(row, col, str(value))

        # Create a sizer to manage layout
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.grid, 1, wx.EXPAND)
        self.panel.SetSizer(sizer)

        self.Show()

# Function that loads the csv file data into the DataViewer class
def show_excel_file(csv_file_path):
    try:
        # Load the first 10 rows of the CSV file into a DataFrame
        data = pd.read_csv(csv_file_path)

        # Open the data in a wxPython frame
        app = wx.App(False)
        frame = DataViewer(data)
        app.MainLoop()

    except FileNotFoundError:
        print("CSV file not found. Please select a valid CSV file.")

def append_to_training_data(csv_file_path):
    file_path = filedialog.askopenfilename(filetypes=[
        ("CSV Files", "*.csv"),
        ("Advanced Record Definition Editor Files", "*.are"),
        ("AS400 Files", "*.as400"),
        ("dBASE Files", "*.dbf"),
        ("Microsoft Access Files", "*.mdb *.accdb"),
        ("Microsoft Excel Files", "*.xls *.xlsx"),
        ("ODBC Files", "*.odbc"),
        ("Print Report Files", "*.pr"),
        ("Adobe PDF Files", "*.pdf"),
        ("SAP/AIS Files", "*.sap"),
        ("Text Files", "*.txt"),
        ("XML Files", "*.xml"),
    ])

    if file_path:

        try:
            # Open the common training data CSV file in append mode
            with open(file_path, 'a', newline='') as training_data:
                writer = csv.writer(training_data)

                # Open the selected CSV file
                with open(csv_file_path, 'r') as selected_csv:
                    reader = csv.reader(selected_csv)

                    # Iterate through rows in the selected CSV and append them to the training data
                    for row in reader:
                        writer.writerow(row)
        except pd.errors.EmptyDataError:
            print("The selected CSV file is empty or has no valid data.", csv_file_path)
        except pd.errors.ParserError:
            print("Error parsing the CSV file. Check the file format and structure.")
        except Exception as e:
            print(f"An error occurred: {str(e)}")

def open_info_dialog():
    # Explanation text
    explanation_text = """
    Benford's Law Analysis:

        A simple python script that takes a CSV and applies Benford's Law to it, where in any financial data typically the leading number 1 appears approx. 30 percent of the time, with a sloping distribution down to 9 that should appear approx. 5 percent of the time.
        The basic theory is that when someone commits fraud, they do not do it in increments of $10, $20, $100, or $200, instead they do it in $80 or $900 increments. Therefor, if the higher numbers appear more often than the large number average distribution, there is a chance fraud is at play.

    Gaussian Distribution Analysis:

        Gives a normal distribution based on the total spent on each Item by each Rep. If there is an abnormally large or small transaction it is made clearly visible for the user to see. WIP: Plots a normal distribution for each item group, with first and second std dev lines plotted to easily pick out outliers.
        Select 3 columns from the drop down menus, an account description (or item category), the transaction description (what item/service was purchased) and the transaction column (total spent on item).

    Transaction Analysis:

     A transctional analytics tool that takes 3 variables from a csv: Which Rep bought an item, what Item was bought, and the Total amount spent on an item. if a person bought the same item multiple times, the graph will sum them but also show a partition to display the different transactions. The graph is interactive, so specific items can be viewed together with others or on their own, and the graph can be zoomed in and out to see all parts of the data or just specific parts the user wants to focus on.
        Select 3 columns from the drop down menus, an account description (or item category), the transaction description (what item/service was purchased) and the transaction column (total spent on item).

    Finding Duplicates:
        Finds duplicate transactions in the data based off of if the 3 columns chosen by the user match with any of the other data, renders it in an excel-like grid in a seperate modal and saves them to a new csv.

    Field Statistics:
        Automatic stats that run when the file is selected such as: number of records, largest value, smallest value.
    """
    messagebox.showinfo("Analysis Explanations", explanation_text)

# Create and configure widgets
label_instruction = ttk.Label(root, text="Select a CSV file:")
entry_file_path = tk.Entry(root, width=60)
button_browse = ttk.Button(root, text="Browse Files", command=browse_file)
button_process_benfords_law = ttk.Button(root, text="Run Benford's Law", command=process_benfords_law)
button_process_gaussian = ttk.Button(root, text="Run Gaussian Distribution", command=lambda: process_gaussian_distribution(entry_file_path.get(), var_col1.get(), var_col3.get(), checkbox_var.get()))
button_process_transaction_analysis = ttk.Button(root, text="Run Transaction Analysis", command=lambda: analyze_transactions_from_csv(entry_file_path.get(), var_col1.get(), var_col2.get(), var_col3.get()))
button_show_excel_file = ttk.Button(root, text="Show Excel File", command=lambda: show_excel_file(entry_file_path.get()))
button_find_duplicate_rows = ttk.Button(root, text="Find Duplicates", command=lambda: find_duplicate_rows(entry_file_path.get(), var_col1.get(), var_col2.get(), var_col3.get()))
info_button = ttk.Button(root, text="Info", command=open_info_dialog)
checkbox = ttk.Checkbutton(root, text="Exclude Negative Values", variable=checkbox_var)
add_to_training_data = ttk.Button(root, text="Add to Training Data", command=lambda: append_to_training_data(entry_file_path.get()) )

# Arrange widgets in the layout using grid
label_instruction.grid(row=0, column=0, columnspan=2, pady=10)
info_button.grid(row=0, column=2, padx=1, pady=1)
entry_file_path.grid(row=1, column=0, columnspan=2, padx=1, pady=1)
button_browse.grid(row=1, column=2, padx=1, pady=1)
button_process_benfords_law.grid(row=2, column=2, padx=1, pady=1)
checkbox.grid(row=3, column=1, padx=1, pady=1)
button_process_gaussian.grid(row=3, column=2, padx=1, pady=1)
button_process_transaction_analysis.grid(row=4, column=2, padx=1, pady=1)
button_find_duplicate_rows.grid(row=5, column=2, columnspan=2, padx=1, pady=1)
button_show_excel_file.grid(row=6, column=2, padx=1, pady=1)
add_to_training_data.grid(row=7, column=2, padx=1, pady=1)

# Start the Tkinter main loop
root.mainloop()


from flask import Blueprint, session, current_app, url_for, redirect

auth = Blueprint('auth', __name__)

@auth.route('/login')
def login():
    google = current_app.oauth.create_client(
        'google')  # create the google oauth client
    redirect_uri = url_for('auth.authorize', _external=True)
    return google.authorize_redirect(redirect_uri)

@auth.route('/authorize')
def authorize():
    google = current_app.oauth.create_client(
        'google')  # create the google oauth client
    # Access token from google (needed to get user info)
    token = google.authorize_access_token()
    # userinfo contains stuff u specificed in the scrope
    resp = google.get('userinfo')
    user_info = resp.json()
    # uses openid endpoint to fetch user info
    user = current_app.oauth.google.userinfo()
    # Here you use the profile/user data that you got and query your database find/register the user
    # and set ur own data in the session not the profile from google
    session['profile'] = user_info
    # make the session permanant so it keeps existing after broweser gets closed
    session.permanent = True
    return redirect('/')

@auth.route('/logout')
def logout():
    for key in list(session.keys()):
        session.pop(key)
    return redirect('/')


import requests
import json

url = 'http://127.0.0.1:8000/crud_api/'

def get_data(id=None):
    data = {}
    if id is not None:
        data = {'id':id}
    json_data = json.dumps(data)
    r = requests.get(url=url,data=json_data)
    data = r.json()
    print(data)

# create record
def record_insert():
    data= {
        'uid': 105,
        'name':'priya',
        'mail':'priya.rai@gmail.com',
        'phone':'8566881104',
        'add1':'120 sardar nagar limbayat',
        'add2':'udhna',
        'city':'surat',
    }
    j_data = json.dumps(data)
    r = requests.post(url= url,data=j_data)
    data = r.json()
    print(data)

# update record
def partial_record_update():
    partial_data= {
        'uid': 104,
        'phone':'8866884401',
    }
    j_data = json.dumps(partial_data)
    r = requests.patch(url= url,data=j_data)
    data = r.json()
    print(data)

def full_record_update():
    full_data= {
        'uid': 104,
        'name':'rahul',
        'mail':'rahul@gmail.com',
        'phone':'9228238777',
        'add1':'120 sardar nagar limbayat',
        'add2':'udhna',
        'city':'surat',
    }
    j_data = json.dumps(full_data)
    r = requests.put(url= url,data=j_data)
    data = r.json()
    print(data)

def record_delete():
    partial_data= {'uid': 104}
    j_data = json.dumps(partial_data)
    r = requests.delete(url= url,data=j_data)
    data = r.json()
    print(data)


import argparse
import numpy as np
import sys,os,time
import void_library as VL
import readgadget
import MAS_library as MASL
import h5py

# read the first and last realization to identify voids
parser = argparse.ArgumentParser(description="This code identifies voids from the realization first to the realization last in the cosmology cosmo")
parser.add_argument("first", help="first realization number", type=int)
parser.add_argument("last",  help="last  realization number", type=int)
parser.add_argument("cosmo", help="folder with the realizations")
args = parser.parse_args()

# This routine find the voids and save them to a file
def find_voids(snapshot, ptypes, grid, MAS, do_RSD, axis, threshold, Radii,
               threads1, threads2, fout):

    # read snapshot header and obtain BoxSize and redshift
    head     = readgadget.header(snapshot)
    BoxSize  = head.boxsize/1e3  #Mpc/h
    redshift = head.redshift

    # compute density field
    delta = MASL.density_field_gadget(snapshot, ptypes, grid, MAS, do_RSD, axis)
    delta /= np.mean(delta, dtype=np.float64);  delta -= 1.0

    # identify voids
    V = VL.void_finder(delta, BoxSize, threshold, Radii,
                       threads1, threads2, void_field=False)

    # void properties and void size function
    void_pos    = V.void_pos
    void_radius = V.void_radius
    VSF_R       = V.Rbins     # bins in radius
    VSF         = V.void_vsf  # void size function

    parameters = np.array([grid, MAS, '%s'%ptypes, threshold, '%s'%Radii])

    # save the results to file
    f = h5py.File(fout, 'w')
    f.create_dataset('grid',          data=grid)
    f.create_dataset('MAS',           data=MAS)
    f.create_dataset('ptype',         data=ptypes)
    f.create_dataset('threshold',     data=threshold)
    f.create_dataset('Radii_input',   data=Radii)
    #f.create_dataset('parameters',    data=parameters)
    f.create_dataset('pos',           data=void_pos)
    f.create_dataset('radius',        data=void_radius)
    f.create_dataset('VSF_Rbins',     data=VSF_R)
    f.create_dataset('VSF',           data=VSF)
    f.close()

#root = '/simons/scratch/fvillaescusa/pdf_information/%s'%args.cosmo
root = '/projects/QUIJOTE/Snapshots/%s'%args.cosmo
################################# INPUT ######################################
# density field parameters
grid    = 768
MAS     = 'PCS'
do_RSD  = False
axis    = 0
snapnum = 4

# void finder parameters
threshold = -0.5
Radii     = np.array([41, 39, 37, 35, 33, 31, 29, 27, 25, 23, 21, 19, 17, 15, 13,
                      11, 9, 7, 5], dtype=np.float32)*1000.0/768
threads1  = 20
threads2  = 4

#root_out = '/simons/scratch/fvillaescusa/pdf_information/Voids/%s'%args.cosmo
root_out = '/projects/QUIJOTE/Voids/%s'%args.cosmo
##############################################################################

z_dict = {4:0, 3:0.5, 2:1, 1:2, 0:3}

# create output folder if it does not exists
if not(os.path.exists(root_out)):  os.system('mkdir %s'%root_out)

# do a loop over the different realizations
for i in range(args.first, args.last):

    # loop over standard and paired fixed simulations
    for prefix in ['%d'%i, 'NCV_0_%d'%i, 'NCV_1_%d'%i]:

        # find the name of the snapshot
        snapshot = '%s/%s/snapdir_%03d/snap_%03d'%(root, prefix, snapnum, snapnum)
        if not(os.path.exists(snapshot+'.0')) and not(os.path.exists(snapshot+'.0.hdf5')):
            continue

        # find output name and create void folder if doesnt exists
        output_folder = '%s/%s/'%(root_out, prefix)
        if not(os.path.exists(output_folder)):  os.system('mkdir %s'%output_folder)

        # find output file name
        fout = '%s/void_catalogue_m_z=%s.hdf5'%(output_folder, z_dict[snapnum])
        if not(os.path.exists(fout)):

            # for massive neutrinos identify voids in the CDM+baryons field
            if args.cosmo in ['Mnu_p', 'Mnu_pp', 'Mnu_ppp']:

                # identify voids in total matter
                find_voids(snapshot, [-1], grid, MAS, do_RSD, axis,
                           threshold, Radii, threads1, threads2, fout)

                # identify voids in CDM+baryons
                fout = '%s/void_catalogue_c_z=%s.hdf5'%(output_folder, z_dict[snapnum])
                if not(os.path.exists(fout)):
                    find_voids(snapshot, [1], grid, MAS, do_RSD, axis,
                               threshold, Radii, threads1, threads2, fout)

            else:

                # identify voids
                find_voids(snapshot, [1], grid, MAS, do_RSD, axis,
                           threshold, Radii, threads1, threads2, fout)


from Homework_3.GameController import GameController

def main():
    while True:
        controller = GameController()
        controller.play_game()
        answer = input("Желаете сыграть еще раз?? (y/n): ")
        if answer.lower() != 'y':
            break

if __name__ == '__main__':
    main()


import sys

from rosalind_utils import get_rosalind_data

def _get_dominant_allele_probability(k, m, n):
    '''
    Compute the probability that in a population of k homozygous dominant, m heterozygous, and n homozygous recessive
    (i.e. k + m + n total population), two random selections will produce offspring with at least one dominant allele

    Assume k, m, n are all positive semidefinite
    '''
    total_pop = k + m + n
    prob_two_dominant = (k/total_pop)*((k-1)/(total_pop-1))
    # can choose heterozygous individual first or second, then homozygous, so just multiply by factor of two
    # (and sim. later)
    prob_one_dominant_one_hetero = 2*(k/total_pop)*(m/(total_pop-1))
    prob_one_dominant_one_recessive = 2*(k/total_pop)*(n/(total_pop-1))
    prob_two_hetero = (m/total_pop)*((m-1)/(total_pop-1))
    prob_one_hetero_one_recessive = 2*(m/total_pop)*(n/(total_pop-1))
    prob_two_recessive = (n/total_pop)*((n-1)/(total_pop-1))

    # draw Mendelian squares for this logic
    # 2 homoz. dominant individuals always produce dominant offspring, so multiply by factor 1 for this combo
    # 1 homoz. dominant 1 heteroz. produce dominant offspring every time because of GG in dom. (GG x Gg)
    # 1 homoz. dominant 1 homoz. recessive produce dominant offspring every time because of GG in dom. (GG x gg)
    # two heterozygous individuals have a dominant allele in 3/4 of combinations (Gg x Gg has a G in 3 of the combos)
    # 1 heteroz. 1 homoz. recessive have dominant allele in 2/3 combinations (Gg x gg has G in 2 of combos)
    # 2 homoz. recessive have 0 combos with dominant allele
    return (1*prob_two_dominant + 1*prob_one_dominant_one_hetero + 1*prob_one_dominant_one_recessive
            + (3/4)*prob_two_hetero + (2/4)*prob_one_hetero_one_recessive
            + (0/4)*prob_two_recessive)

def solve_problem(sequence_data):
    '''
    Assumes only one line in file
    '''
    k, m, n = map(int, sequence_data[0].split())
    prob_dominant_allele = _get_dominant_allele_probability(k, m, n)

    return prob_dominant_allele

if __name__ == '__main__':
    args = list(sys.argv)
    if len(args) != 2:
        raise ValueError('Must provide a single argument which is the data file path')
    solve_problem(get_rosalind_data(args[-1]))


#region: Import libraries and other setup
import os
import time, random

from flask import Flask, render_template
from flask_socketio import SocketIO, send, emit, join_room, leave_room

app = Flask(__name__)
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
socketio = SocketIO(app)

#endregion

#region: Variables
all_messages = [] # list of lists with dictionaries (messages) on the server
all_rooms = [] # list with all rooms on the server
room_messages = [] # list with messages from a specific room
dict_message = {} # dictionary with one message data

#endregion

#region: Route events
@app.route("/")
def index():
    return render_template("index.html", all_rooms=all_rooms)

#endregion

#region: Socketio events

@socketio.on("add room") # when a user sends an "add room" event
def add_room (data):
    all_rooms.append (data["new_room_name"])
    all_messages.append([]) # add an empty sub list
    emit("all rooms", all_rooms, broadcast=True)

@socketio.on('join') # when a user joins a room
def join (data):

    room = data["room"]
    if room in all_rooms:

        join_room(room) # join room function
        room_messages = all_messages[all_rooms.index(room)] # get messages from this room
        emit('switch room name', room) # send user new room name with event to show it on the page
        emit("room messages", room_messages) # send event with json passing up to 100 messages the user
        send({"msg":data["username"] + " has joined the -" + room + "- channel."}, room=room) # send all users in the room this notification of the join
        print(f'\n\n joined room - {room} \n\n') # print to console this message with room

@socketio.on('leave')
def leave(data):

    leave_room(data['room'])
    send({'msg':data['username'] + " has left the -" + data['room'] + "- channel."}, room=data['room'])

@socketio.on('delete_message')
def delete(data):

    dict_message = data["message"]
    room = data["room"]
    room_messages = all_messages[all_rooms.index(room)] # get list of messages from that room
    i = room_messages.index(dict_message) # search list and find index of the message
    print(f'\n\n deleted messages index - {dict_message, i} \n\n') # for debugging
    all_messages[all_rooms.index(room)].pop(i) # remove the message
    emit('delete message', dict_message, room = room) # send event to all users in the room to delete the message

@socketio.on('new_message')
def message(data):
    """Broadcast messages"""

    msg = data["msg"]
    username = data["username"]
    room = data["room"]
    time_stamp = time.strftime('%b-%d %I:%M:%S %p', time.localtime()) # Set timestamp
    msg_id = random.randint(1,1000000) # generate a random number to be a msg ID
    dict_message = {"username": username, "msg": msg, "time_stamp": time_stamp, "msg_id": msg_id} # Dictionary with message data

    # check if server already has 100 msgs for this room and remove the earliest one
    if len(all_messages[all_rooms.index(room)]) == 100:
        all_messages[all_rooms.index(room)].pop(0)

    all_messages[all_rooms.index(room)].append(dict_message) # add message to sub list (the room)
    print(f'\n\n all messages - {room, all_messages, len(all_messages[all_rooms.index(room)])} \n\n') # for debugging - print to console this messate with room
    send(dict_message, room=room) # send message

#endregion

#region: New way of initialization
if __name__ == '__main__':
    socketio.run(app, debug=True)
#endregion


import os
_path = 'E:/My_MA/GazeMiningDataset/Dataset_stimuli/amazon/shots/gaze-mask/'
for imgName in os.listdir(_path):
    iterName = os.path.splitext(imgName)[0]
    photoNameList = iterName.split('_')
    numberName = photoNameList[len(photoNameList)-2]
    number = numberName.split('.')[0]
    reName = ''
    count = 0
    for iter in photoNameList:
        if count == 2:
            reName = reName + '_' + number
        elif count == 0:
            reName = iter
        else:
            reName = reName + '_' + iter
        count = count + 1
    os.rename(_path + imgName,reName+'.png')


n = int(input())
L = []
for _ in range(n):
    x,y,c = map(int,input().split())
    L.append((x,y,c))

l,r = 0,10**10

while r-l>1e-7:
    m = (r+l)/2
    llx = -10**10
    rrx = 10**10
    lly = -10**10
    rry = 10**10
    for x,y,c in L:
        tmp = m/c
        llx = max(llx,x-tmp)
        rrx = min(rrx,x+tmp)
        lly = max(lly,y-tmp)
        rry = min(rry,y+tmp)
    if llx<=rrx and lly<=rry:
        r = m
    else:
        l = m
print(r)


from pathlib import Path

import h5py
import numpy as np

from .polarview import PolarView

from hexrdgui.calibration.utils.maud_headers import header0, header, block_hdr
from hexrdgui.constants import PolarXAxisType, ViewType
from hexrdgui.create_hedm_instrument import create_hedm_instrument
from hexrdgui.hexrd_config import HexrdConfig
from hexrdgui.overlays import update_overlay_data
from hexrdgui.utils.conversions import tth_to_q

def polar_viewer():
    return InstrumentViewer()

class InstrumentViewer:

    def __init__(self):
        self.type = ViewType.polar
        self.instr = create_hedm_instrument()

        self.draw_polar()

    @property
    def all_detector_borders(self):
        return self.pv.all_detector_borders

    @property
    def angular_grid(self):
        return self.pv.angular_grid

    @property
    def raw_img(self):
        return self.pv.raw_img

    @property
    def raw_mask(self):
        return self.pv.raw_mask

    @property
    def snipped_img(self):
        return self.pv.snipped_img

    @property
    def img(self):
        return self.pv.img

    @property
    def snip_background(self):
        return self.pv.snip_background

    def update_angular_grid(self):
        self.pv.update_angular_grid()

    def update_image(self):
        self.pv.generate_image()

    def reapply_masks(self):
        self.pv.reapply_masks()

    def draw_polar(self):
        """show polar view of rings"""
        self.pv = PolarView(self.instr)
        self.pv.warp_all_images()

        '''
        tth_min = HexrdConfig().polar_res_tth_min
        tth_max = HexrdConfig().polar_res_tth_max
        eta_min = HexrdConfig().polar_res_eta_min
        eta_max = HexrdConfig().polar_res_eta_max

        self._extent = [tth_min, tth_max, eta_max, eta_min]  # l, r, b, t
        '''
        self._extent = np.degrees(self.pv.extent)  # l, r, b, t

    def update_overlay_data(self):
        update_overlay_data(self.instr, self.type)

    def update_detector(self, det):
        self.pv.update_detector(det)

    def write_image(self, filename='polar_image.npz'):
        filename = Path(filename)

        azimuthal_integration = np.array(
            HexrdConfig().last_unscaled_azimuthal_integral_data)

        if HexrdConfig().polar_x_axis_type == PolarXAxisType.q:
            # Convert to Q
            azimuthal_integration[0] = tth_to_q(azimuthal_integration[0],
                                                self.instr.beam_energy)

        # Re-format the data so that it is in 2 columns
        azimuthal_integration = azimuthal_integration.T

        # Lineout suffixes and their delimiters
        lineout_suffixes = {
            '.csv': ',',
            '.xy': ' ',
        }
        if filename.suffix in lineout_suffixes:
            # Just save the lineout
            delimiter = lineout_suffixes[filename.suffix]

            # Delete the file if it already exists
            if filename.exists():
                filename.unlink()

            np.savetxt(filename, azimuthal_integration, delimiter=delimiter)
            return

        intensities = self.raw_img.data
        intensities[self.raw_mask] = np.nan

        eta, tth = np.degrees(self.angular_grid)

        # Prepare the data to write out
        data = {
            'tth_coordinates': tth,
            'eta_coordinates': eta,
            'q_coordinates': tth_to_q(tth, self.instr.beam_energy),
            'intensities': intensities,
            'extent': self._extent,
            'azimuthal_integration': azimuthal_integration,
        }

        if self.snip_background is not None:
            # Add the snip background if it was used
            data['snip_background'] = self.snip_background

        # Add visible polar mask data if we have any
        for name, mask in HexrdConfig().masks.items():
            if name not in HexrdConfig().visible_masks:
                continue

            data[f'mask_{name}'] = mask

        # Delete the file if it already exists
        if filename.exists():
            filename.unlink()

        # Check the file extension
        if filename.suffix.lower() == '.npz':
            # If it looks like npz, save as npz
            np.savez(filename, **data)
        else:
            # Default to HDF5 format
            with h5py.File(filename, 'w') as f:
                for key, value in data.items():
                    f.create_dataset(key, data=value)

    def write_maud(self, filename='polar_to_maud.esg'):
        filename = Path(filename)

        with open(filename, 'w') as fid:
            eta_vec = np.degrees(self.angular_grid[0])
            intensities = self.img
            first_block = True
            for i, eta in enumerate(np.average(eta_vec, axis=1).flatten()):
                if np.all(np.isnan(intensities[i])):
                    # Skip this block
                    continue

                if first_block:
                    hstr = header0 % (i, np.linalg.norm(self.pv.tvec_s), eta)
                    first_block = False
                else:
                    hstr = header % (i, eta)

                fid.write(hstr)
                fid.write(block_hdr)
                tth = HexrdConfig().last_unscaled_azimuthal_integral_data[0]
                for rho, inten in zip(tth, intensities[i]):
                    if np.isnan(inten):
                        continue

                    vals = f' {rho}  {inten}\n'
                    fid.write(vals)


'''
Independent Arbitrary Cascade (IAC) is a independent cascade model with arbitrary
 propagation probabilities.
'''

from copy import deepcopy
import random, multiprocessing, os, math, json
import networkx as nx
import matplotlib.pylab as plt

def uniformEp(G, p = .01):
    '''
    Every edge has the same probability p.
    '''
    if type(G) == type(nx.DiGraph()):
        Ep = dict(list(zip(G.edges(), [p]*len(G.edges()))))
    elif type(G) == type(nx.Graph()):
        Ep = dict()
        for (u, v) in G.edges():
            Ep[(u, v)] = p
            Ep[(u, v)] = p
    else:
        raise ValueError("Provide either nx.Graph or nx.DiGraph object")
    #Convert Ep into nx.DiGraph type

    P = nx.DiGraph()
    for key in Ep:
        u = key[0]
        v = key[1]
        P.add_edge(u,v, weight = Ep[key])
    #return Ep
    return P

def randomEp(G, maxp):
    '''
    Every edge has random propagation probability <= maxp <= 1
    '''
    assert maxp <= 1, "Maximum probability cannot exceed 1."
    Ep = dict()
    if type(G) == type(nx.DiGraph()):
        for v1,v2 in G.edges():
            p = random.uniform(0, maxp)
            Ep[(v1,v2)] = p
    elif type(G) == type(nx.Graph()):
        for v1,v2 in G.edges():
            p = random.uniform(0, maxp)
            Ep[(v1,v2)] = p
            Ep[(v2,v1)] = p
    else:
        raise ValueError("Provide either nx.Graph or nx.DiGraph object")

    P = nx.DiGraph()
    for key in Ep:
        u = key[0]
        v = key[1]
        P.add_edge(u,v, weight = Ep[key])
    #return Ep
    return P

def random_from_range (G, prange):
    '''
    Every edge has propagation probability chosen from prange uniformly at random.
    '''
    for p in prange:
        if p > 1:
            raise ValueError("Propagation probability inside range should be <= 1")
    Ep = dict()
    if type(G) == type(nx.DiGraph()):
        for v1,v2 in G.edges():
            p = random.choice(prange)
            Ep[(v1,v2)] = p
    elif type(G) == type(nx.DiGraph()):
        for v1,v2 in G.edges():
            p = random.choice(prange)
            Ep[(v1,v2)] = p
            Ep[(v2,v1)] = p

    P = nx.DiGraph()
    for key in Ep:
        u = key[0]
        v = key[1]
        P.add_edge(u,v, weight = Ep[key])
    #return Ep
    return P

# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
def chunks(lst, n):
    size = int(math.ceil(float(len(lst))/n))
    return [lst[i:i + size] for i in range(0, len(lst), size)]

def degree_categories(G, prange):
    '''
    Every edge has propagation probability chosen from prange based on degree of a node.
    '''
    for p in prange:
        if p > 1:
            raise ValueError("Propagation probability inside range should be <= 1")
    Ep = dict()

    d = {v: sum([G[v][u]["weight"] for u in G[v]]) for v in G}
    sorted_d = chunks(sorted(iter(d.items()), key = lambda __degree: __degree[1]), len(prange))
    sorted_p = sorted(prange)
    categories = list(zip(sorted_p, sorted_d))
    dp = dict()
    for c in categories:
        p, nodes = c
        for (v, _) in nodes:
            dp[v] = p

    if type(G) == type(nx.DiGraph()):
        for v1,v2 in G.edges():
            Ep[(v1,v2)] = dp[v1]
    elif type(G) == type(nx.DiGraph()):
        for v1,v2 in G.edges():
            Ep[(v1,v2)] = dp[v2]
            Ep[(v2,v1)] = dp[v1]

    P = nx.DiGraph()
    for key in Ep:
        u = key[0]
        v = key[1]
        P.add_edge(u,v, weight = Ep[key])
    #return Ep
    return P

def weightedEp(G):
    '''
    Every incoming edge of v has propagation probability equals to 1/deg(v)
    '''
    Ep = dict()
    for v in G:
        in_edges = G.in_edges([v])
        degree = sum([G[u][v]["weight"] for (u, _) in in_edges])
        for edge in in_edges:
            Ep[edge] = 1.0/degree

    P = nx.DiGraph()
    for key in Ep:
        u = key[0]
        v = key[1]
        P.add_edge(u,v, weight = Ep[key])
    #return Ep
    return P

def runIACmodel (G, S, Ep):
    ''' Runs independent cascade model.
    Input: G -- networkx graph object
    S -- initial set of vertices
    Ep -- propagation probability graph
    Output: T -- resulted influenced set of vertices (including S)
    '''
    T = deepcopy(S) # copy already selected nodes
    E = {}

    # ugly C++ version
    i = 0
    while i < len(T):
        for v in G[T[i]]: # for neighbors of a selected node
            if v not in T: # if it wasn't selected yet
                w = G[T[i]][v]['weight'] # count the number of edges between two nodes
                p = Ep[(T[i],v)] # propagation probability
                if random.random() <= 1 - (1-p)**w: # if at least one of edges propagate influence
                    # print T[i], 'influences', v
                    T.append(v)
                    E[(T[i], v)] = 1
        i += 1

    return len(T), T, E

def runIAC (G, S, Ep):
    ''' Runs independent arbitrary cascade model.
    Input: G -- networkx graph object
    S -- initial set of vertices
    Ep -- propagation probabilities
    Output: T -- resulted influenced set of vertices (including S)

    NOTE:
    Ep is a dictionary for each edge it has associated probability
    If graph is undirected for each edge (v1,v2) with probability p,
     we have Ep[(v1,v2)] = p, Ep[(v2,v1)] = p.
    '''
    T = deepcopy(S) # copy already selected nodes

    # ugly C++ version
    i = 0
    while i < len(T):
        for v in G[T[i]]: # for neighbors of a selected node
            if v not in T: # if it wasn't selected yet
                w = G[T[i]][v]['weight'] # count the number of edges between two nodes
                p = Ep[(T[i],v)] # propagation probability
                if random.random() <= 1 - (1-p)**w: # if at least one of edges propagate influence
                    # print T[i], 'influences', v
                    T.append(v)
        i += 1
    return T

def avgIAC (G, S, Ep, I):
    '''
    Input:
        G -- undirected graph
        S -- seed set
        Ep -- propagation probabilities
        I -- number of iterations
    Output:
        avg -- average size of coverage
    '''
    avg = 0
    for i in range(I):
        avg += float(len(runIAC(G,S,Ep)))/I
    return avg

def findCC(G, Ep):
    # remove blocked edges from graph G
    E = deepcopy(G)
    edge_rem = [e for e in E.edges() if random.random() < (1-Ep[e])**(E[e[0]][e[1]]['weight'])]
    E.remove_edges_from(edge_rem)

    # initialize CC
    CC = dict() # number of a component to its members
    explored = dict(list(zip(E.nodes(), [False]*len(E))))
    c = 0
    # perform BFS to discover CC
    for node in E:
        if not explored[node]:
            c += 1
            explored[node] = True
            CC[c] = [node]
            component = list(E[node].keys())
            for neighbor in component:
                if not explored[neighbor]:
                    explored[neighbor] = True
                    CC[c].append(neighbor)
                    component.extend(list(E[neighbor].keys()))
    return CC

def findL(CCs, T):
    # find top components that can reach T activated nodes
    sortedCCs = sorted([(len(dv), dk) for (dk, dv) in CCs.items()], reverse=True)
    cumsum = 0 # sum of top components
    L = 0 # current number of CC that achieve T
    # find L first
    for length, numberCC in sortedCCs:
        L += 1
        cumsum += length
        if cumsum >= T:
            break
    return L, sortedCCs

def findCCs_size_distribution(G, Ep, T):
    CCs = findCC(G, Ep)
    L, sortedCCs = findL(CCs, T)
    from itertools import groupby
    histogram = [(s, len(list(group))) for (s, group) in groupby(sortedCCs, key = lambda size__: size__[0])]

    bluedots = 1
    acc_size = 0
    for (size, number) in histogram:
        acc_size += size
        if acc_size < T:
            bluedots += 1
        else:
            break

    return histogram, bluedots, L, len(CCs)

def findLrangeforTrange (G, Ep, Trange):
    Lrange = []
    CCs = findCC(G, Ep)
    for T in Trange:
        L, _ = findL(CCs, T)
        Lrange.append(L)
    return Lrange, len(CCs)

if __name__ == '__main__':
    import time
    start = time.time()

    # read in graph
    # G = nx.DiGraph()
    # with open('../../graphdata/hep.txt') as f:
    #     n, m = f.readline().split()
    #     for line in f:
    #         try:
    #             u, v = map(int, line.split())
    #         except ValueError:
    #             continue
    #         try:
    #             G[u][v]["weight"] += 1
    #             G[v][u]["weight"] += 1
    #         except:
    #             G.add_edge(u, v, weight=1)
    #             G.add_edge(v, u, weight=1)
    # print 'Built graph G'
    # print time.time() - start
    #
    #
    # nx.write_gpickle(G, "../../graphs/hep.gpickle")
    # print 'Wrote graph G'
    # print time.time() - start
    G = nx.read_gpickle("../../graphs/hep.gpickle")
    print('Read graph G')
    print(time.time() - start)

    DROPBOX = "/home/sergey/Dropbox/Influence Maximization/"

    random.seed(1)

    # time2probability = time.time()
    # prange = [.01, .02, .04, .08]
    # Ep = random_from_range(G, prange)
    # print 'Built probabilities Ep'
    # print time.time() - time2probability
    # model = "MultiValency"
    #
    # write CCs sizes distrbution to file
    # T = 500
    # histogram, bluedots, L, TotalCCs = findCCs_size_distribution(G, Ep, T)
    # with open("plotdata/CCs_sizes_Multivalency1.txt", "w+") as fp:
    #     print >>fp, bluedots
    #     print >>fp, T
    #     print >>fp, L
    #     print >>fp, TotalCCs
    #     print >>fp, json.dumps(histogram)
    #
    # write TvsL to file
    # Trange = range(1, 3001, 50)
    # Lrange, TotalCCs = findLrangeforTrange(G, Ep, Trange)
    # with open("plotdata/LvsT_%s.txt" %model, "w+") as fp:
    #     print >>fp, json.dumps(Trange)
    #     print >>fp, json.dumps(Lrange)
    #     print >>fp, TotalCCs
    # with open(DROPBOX + "plotdata/LvsT_%s.txt" %model, "w+") as fp:
    #     print >>fp, json.dumps(Trange)
    #     print >>fp, json.dumps(Lrange)
    #     print >>fp, TotalCCs

    # with open("Ep_hep_range1.txt", "w+") as f:
    #     for key, value in Ep.iteritems():
    #         f.write(str(key[0]) + " " + str(key[1]) + " " + str(value) + os.linesep)
    #
    # time2probability = time.time()
    # prange = [.01, .02, .04, .08]
    # Ep = degree_categories(G, prange)
    # print 'Built probabilities Ep'
    # print time.time() - time2probability
    # model = "Categories"
    #
    # time2probability = time.time()
    # Ep = randomEp(G, .1)
    # print 'Built probabilities Ep'
    # print time.time() - time2probability
    # model = "Random"
    #
    # time2probability = time.time()
    # Ep = uniformEp(G, .01)
    # print 'Built probabilities Ep'
    # print time.time() - time2probability
    #
    # time2probability = time.time()
    # Ep = weightedEp(G)
    # print 'Built probabilities Ep'
    # print time.time() - time2probability
    #
    # import json
    # coverage2length = [[0,0]]
    # with open("plotdata/rawCCWPforDirect2.txt") as f:
    #     for line in f:
    #         [(cov, S)] = json.loads(line).items()
    #         coverage2length.append([len(S), int(cov)])
    #
    # coverage2length.sort(key=lambda (l,_): l)
    #
    # with open("plotdata/plotReverseCCWPforReverse2_v2.txt", "w+") as f:
    #     json.dump(coverage2length, f)

    console = []


from torch.utils.data import random_split
import matplotlib.pyplot as plt
from data import get_device, to_device, SepsisDataset, RandomDataset
from net import FederatedNet
from client import Client
from args import parse_args
import plotext as plt

def main():
    args = parse_args()
    num_clients = args.clients

    if args.sepsis:
        classes = 2
        input_dim = 3

        train_dataset = SepsisDataset(train=True, num_clients=num_clients)
        test_dataset = SepsisDataset(train=False, num_clients=num_clients)

        train_dataset, dev_dataset = random_split(train_dataset, [int(len(train_dataset) * 0.83), int(len(train_dataset) * 0.17)])

        print(f'{len(train_dataset)} {len(dev_dataset)} {len(test_dataset)}')

    elif args.random:
        classes = 3
        input_dim = 4

        train_dataset = RandomDataset(train=True)
        test_dataset = RandomDataset(train=False)

        train_dataset, dev_dataset = random_split(train_dataset, [int(len(train_dataset) * 0.83), int(len(train_dataset) * 0.17)])

        print(f'{len(train_dataset)} {len(dev_dataset)} {len(test_dataset)}')

    else:
        print("Error: please specify data set argument value")

    total_train_size = len(train_dataset)
    total_test_size = len(test_dataset)
    total_dev_size = len(dev_dataset)

    # num_clients = args.clients
    rounds = args.epochs
    batch_size = args.batch
    epochs_per_client = 3
    learning_rate = 2e-2

    device = get_device()

    examples_per_client = total_train_size // num_clients
    client_datasets = random_split(train_dataset, [min(i + examples_per_client,
            total_train_size) - i for i in range(0, total_train_size, examples_per_client)])
    clients = [Client('client_' + str(i), client_datasets[i]) for i in range(num_clients)]

    global_net = to_device(FederatedNet(input_dim, classes), device)

    history = []
    round_acc = []

    for i in range(rounds):
        print('Start Round {} ...'.format(i + 1))
        curr_parameters = global_net.get_parameters()
        new_parameters = dict([(layer_name, {'weight': 0, 'bias': 0}) for layer_name in curr_parameters])
        for client in clients:
            client_parameters = client.train(curr_parameters, device, input_dim, classes, epochs_per_client, learning_rate, batch_size)
            fraction = client.get_dataset_size() / total_train_size
            for layer_name in client_parameters:
                new_parameters[layer_name]['weight'] += fraction * client_parameters[layer_name]['weight']
                new_parameters[layer_name]['bias'] += fraction * client_parameters[layer_name]['bias']
        global_net.apply_parameters(new_parameters)

        train_loss, train_acc = global_net.evaluate(train_dataset, device, batch_size)
        dev_loss, dev_acc = global_net.evaluate(dev_dataset, device, batch_size)
        print('After round {}, train_loss = {}, dev_loss = {}, dev_acc = {}\n'.format(i + 1, round(train_loss, 4),
                round(dev_loss, 4), round(dev_acc, 4)))
        history.append((train_loss, dev_loss))
        round_acc.append(train_acc)

    plt.plot(round_acc)
    plt.theme('matrix')
    plt.plotsize(50, 15)
    plt.show()

if __name__ == '__main__':
    main()


import numpy as np
import cv2
import h5py
import pandas as pd
from scipy.stats import entropy

class FeatureExtractor:
    def __init__(self, batch_size=10000, img_size=(224,224)):
        (self.image_w, self.image_h) = img_size
        self.batch_size = batch_size

        # opencv body detection model
        self.opencv_path = './opencv/'
        self.full_body_detector = cv2.CascadeClassifier(self.opencv_path + 'haarcascade_fullbody.xml')
        self.upper_body_detector = cv2.CascadeClassifier(self.opencv_path + 'haarcascade_upperbody.xml')
        self.lower_body_detector = cv2.CascadeClassifier(self.opencv_path + 'haarcascade_lowerbody.xml')
        self.face_detector = cv2.CascadeClassifier(self.opencv_path + 'haarcascade_frontalface_alt2.xml')

        # size(big, medium, small)
        self.size_degree = (0.55, 0.35)
        # classify color(HSV not RGB)
        self.colors = ((0, 0, 255, "white"),
                      (0, 0, 0, "black"),
                      (0, 0, 120, "grey"),
                      (176, 215, 190, "red"),
                      (23, 151, 220, "yellow"),
                      (35, 132, 139, "khaki"),
                      (105, 85, 180, "blue"),
                      (120, 64, 80, "navy"),
                      (13, 89, 100, "brown"),
                      (15, 113, 180, "beige"))

    # Main
    def extract_feature(self, image_file='temp/images.hdf5', masks_file='temp/masks.hdf5'):
        # check detector loaded
        if self.full_body_detector.empty() or self.upper_body_detector.empty() or self.lower_body_detector.empty() or self.face_detector.empty():
            print('Classifier XML not Loaded!')
            return None

        # load data
        try :
            with h5py.File(image_file, 'r') as f, h5py.File(masks_file, 'r') as m:
                num_X = len(f['image'])

                ##########################################################
                #num_X = 1000
                #self.batch_size = 1000
                self.a = np.zeros((num_X, 3), dtype=np.float32)

                # initialization
                color = np.empty(num_X, dtype=np.dtype("U10"))
                bg_color = np.empty(num_X, dtype=np.dtype("U10"))
                comp = np.zeros(num_X, dtype=np.uint8)
                size = np.zeros(num_X, dtype=np.uint8)
                model_type = np.zeros(num_X, dtype=np.uint8)

                for i in range(0, num_X, self.batch_size):
                    # lazy load data(hdf5)
                    X = f['image'][i:i+self.batch_size]
                    masks = m['mask'][i:i+self.batch_size]
                    print('{}~{} Data Loaded'.format(i, i+len(masks)))

                    # 상품, 배경 색감 추출
                    color[i:i+self.batch_size], bg_color[i:i+self.batch_size] = self.extract_color(X, masks)
                    # 배경의 복잡도
                    comp[i:i+self.batch_size] = self.image_complexity(X, masks)
                    # 상품 크기 비율
                    size[i:i+self.batch_size] = self.estimate_size(masks)
                    # 모델의 노출도
                    model_type[i:i+self.batch_size] = self.detect_body(X)

                df = pd.DataFrame({'id': f['index'], 'color': color, 'bg_color': bg_color, 'bg_complexity': comp, 'size': size, 'model_type': model_type})
                print('Feature Extracted.')
        except :
            df = None
        return df

    def nearest_color(self, x):
        return min(self.colors, key=lambda color: sum((c - x_) ** 2 for c, x_ in zip(color, x)))[-1]

    def extract_color(self, X, masks):
        # reshape
        masks_ = np.reshape(masks, masks.shape[:-1]).astype(np.bool)
        # extract color
        num_X = len(X)
        color = np.zeros(num_X, dtype=np.dtype("U10"))
        bg_color = np.zeros(num_X, dtype=np.dtype("U10"))

        # convert BGR(opencv default) to HSV
        X_ = np.reshape(X, (num_X * self.image_w, self.image_h, 3))
        X_ = cv2.cvtColor(X_, cv2.COLOR_BGR2HSV)
        X_ = np.reshape(X_, X.shape)

        # test
        for i, (x, m) in enumerate(zip(X_, masks_)):
            #print(self.nearest_color(colors, np.mean(X[i][masks[i]], axis=0).astype(int)))
            color[i] = self.nearest_color(np.mean(x[m], axis=0).astype(np.uint8))
            #print(self.nearest_color(colors, np.mean(X[i][np.logical_not(masks[i])], axis=0).astype(int)))
            bg_color[i] = self.nearest_color(np.mean(x[np.logical_not(m)], axis=0).astype(np.uint8))

        print('extract_color Done.')
        return color, bg_color

    # 배경 복잡도 (1: 복잡함 0:안복잡함)
    def image_complexity(self, X, masks):
        # 복잡도 기준
        threshold = 25.0
        # reshape
        # masks = np.reshape(masks, masks.shape[:-1])

        R, G, B = X[:, :, :, 0:1], X[:, :, :, 1:2], X[:, :, :, 2:]

        # compute rg = R - G
        rg = np.absolute(R - G)

        # compute yb = 0.5 * (R + G) - B
        yb = np.absolute(0.5 * (R + G) - B)

        _masks = np.logical_not(masks)
        # compute the mean and standard deviation of both `rg` and `yb` ()
        rb_masked = np.ma.masked_where(_masks, rg)
        (rb_mean, rb_std) = (np.mean(rb_masked, axis=(1, 2, 3)), np.std(rb_masked, axis=(1, 2, 3)))

        yb_masked = np.ma.masked_where(_masks, yb)
        (yb_mean, yb_std) = (np.mean(yb_masked, axis=(1, 2, 3)), np.std(yb_masked, axis=(1, 2, 3)))

        # combine the mean and standard deviations
        std_root = np.sqrt((rb_std ** 2) + (yb_std ** 2))
        mean_root = np.sqrt((rb_mean ** 2) + (yb_mean ** 2))

        # derive the "colorfulness" metric and return it
        complexity = std_root + (0.3 * mean_root)

        # 수치 수정
        print('image_complexity Done.')
        return (complexity >= threshold).astype(np.uint8)

    def estimate_size(self, masks):
        d = np.sqrt(np.sum(masks, axis=(1, 2, 3), dtype=np.uint32) / (masks.shape[1] * masks.shape[2]))
        # big: 2, medium: 1, small: 0
        size = np.zeros_like(d)
        size = np.where(d >= self.size_degree[1], 1, size)
        size = np.where(d >= self.size_degree[0], 2, size)

        print('estimate_size Done.')
        return size

    # 5: 전신 4: 얼굴빼고 전신 3: 상반신 2: 얼굴빼고 상반신 1: 하반신 0: 모델 없음
    def detect_body(self, X):
        # init
        num_X = len(X)
        model_type = np.zeros(num_X, dtype=np.uint8)

        # detect each image
        for i, x in enumerate(X):
            img = cv2.cvtColor(x, cv2.COLOR_RGB2GRAY)
            # full_body_detected
            full_body_detected = len(self.full_body_detector.detectMultiScale(img, 1.01, 3)) > 0
            #upper_body_detected
            upper_body_detected = len(self.upper_body_detector.detectMultiScale(img, 1.01, 3)) > 0
            #lower_body_detected
            lower_body_detected = len(self.lower_body_detector.detectMultiScale(img, 1.01, 3)) > 0

            if full_body_detected and (upper_body_detected or lower_body_detected):
                # 5: 전신(with face)
                if len(self.face_detector.detectMultiScale(img, 1.01, 3)) > 0:
                    model_type[i] = 5
                # 4: 얼굴빼고 전신
                else:
                    model_type[i] = 4
            elif upper_body_detected:
                # 3: 상반신(with face)
                if len(self.face_detector.detectMultiScale(img, 1.01, 3)) > 0:
                    model_type[i] = 3
                # 2: 얼굴빼고 상반신
                else:
                    model_type[i] = 2
            # 1: 하반신
            elif lower_body_detected:
                model_type[i] = 1
            # 0: 모델 없음
            else :
                model_type[i] = 0

        print('detect_body Done.')
        return model_type

if __name__=='__main__':
    item_dir = 'temp/'
    extractor = FeatureExtractor()
    # extract feature
    df = extractor.extract_feature(image_file=item_dir+'images.hdf5', masks_file=item_dir+'masks.hdf5')
    # save csv
    df.to_csv(item_dir + 'features.csv', sep=',', na_rep='NaN', encoding='utf-8')


# input

n = int(input())

# loop

for number in range(1111, 10000):
    units = number % 10
    tens = number // 10 % 10
    hundreds = number // 100 % 10
    thousands = number // 1000
    if n % units == 0 and n % tens == 0 and n % hundreds == 0 and n % thousands == 0:
        print(number)


import torch
import pytorch_lightning as pl
from tokenizers import Tokenizer
from transformers import AdamW

from utils.token_utils import get_vocab_size, get_token_id, get_weights
from utils.utils import fussed_lasso, calc_acc, calculate_mask_percentage, get_pad_id, calc_perplexity, \
    calc_cross_entropy_batch_wise, calc_policy_loss

class LightingReinforceRationalizedLanguageModel(pl.LightningModule):
    '''
    PL wrapper for training a language model together with a rational extractor.
    '''

    def __init__(self, language_model, rational_extractor, tokenizer, hparams=None,
                 sparsity_weight=0.0001,
                 fussed_lasso_weight=0.1):
        super().__init__()
        self.hparams = hparams
        self.language_model = language_model
        self.rational_extractor = rational_extractor
        self.tokenizer = tokenizer

        self.sparsity_weight = sparsity_weight
        self.fussed_lasso_weight = fussed_lasso_weight

        self.log_list = [
            "total_loss", "acc", "total_mask_loss", "mask_mean", "mask_fussed_lasso", "cross_entropy_loss",
            "perplexity",
        ]

        self.pad_token_id = get_pad_id(tokenizer)

        self.freeze_language_model = hparams["freeze_language_model"]

    def forward(self, x, targets, ):

        rational = self.get_rational(x)

        masked_input = rational["masked_input"]

        prediction = self.forward_masked_input(masked_input, targets)
        return {"logits": prediction, **rational, "x": x}

    def get_rational(self, x, greedy=False):
        if greedy:
            print(greedy)
            return self.rational_extractor(x, greedy=greedy)
        else:
            return self.rational_extractor(x)

    def forward_masked_input(self, masked_input, targets):
        ## Concatenate the two together and put through the lstm
        lm_in = torch.cat([masked_input, targets])

        prediction = self.language_model(lm_in)
        return prediction

    def get_scores(self, forward_dict, targets):
        """
        Gets a dict with multiply scores of the give prediction
        prediction
        forward_dict: {logits, mask, chosen_policy}
        """
        targets = targets.long()

        n_targets = targets.shape[0]
        predictions = forward_dict["logits"][-(n_targets + 1):-1]

        total_h_loss = 0
        h_mean = 0
        fussed_lasso_loss = 0

        if "mask" in forward_dict.keys():
            ### Make sure batch first.
            mask = forward_dict["mask"].permute(1, 0).float()
            tokens = forward_dict["x"].permute(1, 0).float()
            h_mean = calculate_mask_percentage(tokens, mask, reduce=False,
                                               pad_id=get_token_id(self.tokenizer, "pad_token"))
            fussed_lasso_loss = fussed_lasso(tokens, mask, reduce=False,
                                             pad_id=get_token_id(self.tokenizer, "pad_token"))
            total_h_loss = self.sparsity_weight * h_mean + self.fussed_lasso_weight * fussed_lasso_loss

        cross_entropy_loss = calc_cross_entropy_batch_wise(predictions, targets, self.tokenizer)
        ##Make sure it is mean per batch

        rewards = cross_entropy_loss + total_h_loss
        vocab_size = get_vocab_size(self.tokenizer)
        perplexity = calc_perplexity(predictions,
                                     targets, self.tokenizer)

        # Get the policy loss. (old one)
        # total_loss = -torch.mean(rewards.detach() * torch.log(forward_dict["chosen_policy"]))
        total_loss = calc_policy_loss(rewards, forward_dict["chosen_policy"])

        if not self.freeze_language_model:
            total_loss += torch.mean(rewards)

        acc = calc_acc(predictions.reshape(-1, vocab_size),
                       targets.flatten(), exclude=self.pad_token_id)

        return {"total_loss": total_loss, "acc": acc, "total_mask_loss": total_h_loss.mean(),
                "mask_mean": h_mean.mean(),
                "mask_fussed_lasso": fussed_lasso_loss.mean(), "cross_entropy_loss": cross_entropy_loss.mean(),
                "perplexity": perplexity}

    def batch_to_out(self, batch):
        rational_in = batch[0].permute(1, 0)
        targets = batch[1].permute(1, 0)

        out = self.forward(rational_in, targets)

        scores = self.get_scores(out, targets)

        return scores

    def training_step(self, batch, batch_idx):

        batch_out = self.batch_to_out(batch)

        self.log_results(batch_out)

        return batch_out["total_loss"]

    @torch.no_grad()
    def validation_step(self, batch, batch_idx):

        batch_out = self.batch_to_out(batch)

        self.log_results(batch_out, prepend="val ")

        return batch_out["total_loss"]

    def complete_dialogues(self, sentences, total_length, with_rational=True, greedy_rationals=True):
        return [self.complete_dialogue(sentence, total_length=total_length, with_rational=with_rational,
                                       greedy_rationals=greedy_rationals) for sentence in
                sentences]

    def complete_dialogue(self, completed_dialogue, total_length=100, with_rational=True,
                          greedy_rationals=True):

        if type(self.tokenizer) == Tokenizer:
            encoding = self.tokenizer.encode(completed_dialogue).ids
        else:
            encoding = self.tokenizer.encode(completed_dialogue)

        all_tokens = torch.tensor(encoding).to(self.device).unsqueeze(1)

        ids_tensor = all_tokens
        rationals = []
        responses = []
        rationalized_input = []
        while (len(ids_tensor)) < total_length:

            if with_rational:
                # Get the rational
                rational = self.rational_extractor(ids_tensor, greedy=greedy_rationals)

                # Map back to tokens
                rational_input = self.tokenizer.decode(rational["masked_input"].long().view(-1).detach().cpu().numpy(),
                                                       skip_special_tokens=False).replace(" #",
                                                                                "").replace(
                    "#", "")
                next_input = rational["masked_input"]
                # The mask
                rationals.append(rational["mask"].flatten())

            else:
                next_input = ids_tensor
                rational_input = self.tokenizer.decode(ids_tensor.long().view(-1).detach().cpu().numpy(),
                                                       skip_special_tokens=False).replace(" #",
                                                                                "").replace(
                    "#", "")
            rationalized_input.append(rational_input)

            # Generate next ids based on the masked input
            # next_ids = self.language_model.generate_next_tokens(next_input, n_tokens=n_rational)
            next_ids = self.language_model.next_utterance(next_input.flatten(),
                                                          get_token_id(self.tokenizer, "sep_token"),
                                                          max_length=20).reshape(-1, 1)
            # next_ids = self.language_model.lm.generate(next_input.reshape(1,-1),
            #                                             eos_token_id=self.tokenizer.sep_token_id,
            #                                             num_beams=5,
            #                                             min_length=5,
            #                                             max_length=1000,#(20+len(next_input)),
            #                                             forced_eos_token_id=self.tokenizer.sep_token_id,
            #                                             ).reshape(-1,1) #TODO keep it to check if it will work in the future
            # Add to all tokens
            all_tokens = torch.cat([all_tokens, next_ids])

            # Map back to the sentence
            responses.append(
                # self.tokenizer.decode(next_ids.reshape(-1).detach().cpu().numpy(), skip_special_tokens=False).replace(" #", "").replace("#", "")
                self.tokenizer.decode(next_ids.reshape(-1).detach().cpu().numpy(), skip_special_tokens=False)
            )

            # Map back to tensor
            ids_tensor = all_tokens

        completed_dialogue = self.tokenizer.decode(all_tokens.reshape(-1).detach().cpu().numpy(),
                                                   skip_special_tokens=False).replace(" #", "").replace("#", "")

        result = {"completed_dialogue": completed_dialogue, "rationals": rationals,
                  "rationalized_input": rationalized_input,
                  "response": responses}

        return result

    def configure_optimizers(
            self,
    ):
        if not self.freeze_language_model:
            parameters = list(self.language_model.parameters()) + list(self.rational_extractor.parameters())
        else:
            parameters = list(self.rational_extractor.parameters())

        optimizer = AdamW(
            parameters,
            lr=self.hparams['learning_rate'])
        return optimizer

    def log_results(self, result, prepend=""):

        for k in self.log_list:
            self.log(prepend + k, result[k], on_step=True, on_epoch=True)


import requests
from bs4 import BeautifulSoup

'''
get job about keyward(ex: python, java, ...) from remoteok.com

all_job=[job1_dict,job2_dict]
job1_dict={
  "title":title,
  "company":company,
  "location":location,
  "introduction":introduction,
  "date":date,
  "link":link
}
'''

HEADERS_DEFAULT = {
    'User-Agent':
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
}

ROK_URL="https://remoteok.io"

def get_jobs(keyward,headers):
  req=requests.get(f"{ROK_URL}/remote-dev+{keyward}-jobs",headers=headers)
  soup=BeautifulSoup(req.text,"html.parser")
  jobcard_list=soup.find_all("tr",{"class":"job"})
  job_list=[]
  for jobcard in jobcard_list:
    #remove closed
    if jobcard.find("td",{"class":"company"}).find("span",{"class":"closed"}) is not None:
      continue
    #introduction
    introduction_list=jobcard.find("td",{"class":"tags"}).find_all('a')
    introduction="develop tool:"
    for info in introduction_list:
      tag=info.text.replace("\xa0","")
      introduction+=f" [{tag}]"
    #location
    try:
      location=jobcard.find("td",{"class":"company"}).find("div",{"class":"location"}).text
    except:
      location=None
    job={
      "title":jobcard.find("td",{"class":"company"}).find("h2",{"itemprop":"title"}).text,
      "company":jobcard.find("td",{"class":"company"}).find("h3",{"itemprop":"name"}).text,
      "location":location,
      "introduction":introduction,
      "date":jobcard.find("td",{"class":"time"}).find("time").text,
      "link":ROK_URL+jobcard.find("td",{"class":"source"}).find("a")["href"]
    }
    job_list.append(job)
  return job_list

def ROK_jobs(keyward,headers=HEADERS_DEFAULT):
  all_jobs=get_jobs(keyward,headers)
  print(f"keyward: {keyward}, website: remoteok, page: 1, extracted")
  return all_jobs


from random import randint
number = randint(1, 10)
guesses = 0

while guesses <3:
    guess = int(input("Guess a number between 1-10! "))
    guesses = guesses + 1
    if guess != number:
        print("Incorrect, try again!")
    elif guess == number:
        print("Correct, well done!")
        print("You guessed the number in {} tries!".format(guesses))
        break
else:
    print("You've run out of guesses!")


from firebase import firebase
import cv2

FBConn = firebase.FirebaseApplication("https://pikachut-8f775-default-rtdb.firebaseio.com/",None)
johnsmith = cv2.imread('johnsmith.jpg')

while True:
    temperature = int(input("What is the temp?"))
    data_to_upload = {
        'Temp':temperature,
        'Name': johnsmith
    }

    result=FBConn.post('/MyTestData/',data_to_upload)

    print(result)


from odoo.tests import common

class TestCrmLocation(common.TransactionCase):
    @classmethod
    def setUpClass(cls):
        super(TestCrmLocation, cls).setUpClass()
        cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True))
        cls.country = cls.env["res.country"].create({"name": "Test country"})
        cls.state = cls.env["res.country.state"].create(
            {
                "name": "Test state",
                "code": "Test state code",
                "country_id": cls.country.id,
            }
        )
        cls.city = cls.env["res.city"].create(
            {
                "name": "Test city",
                "country_id": cls.country.id,
                "state_id": cls.state.id,
            }
        )
        cls.location = cls.env["res.city.zip"].create(
            {"name": "12345", "city_id": cls.city.id}
        )
        cls.lead = cls.env["crm.lead"].create({"name": "Test lead"})
        cls.partner = cls.env["res.partner"].create(
            {
                "name": "Test partner name",
                "state_id": cls.state.id,
                "country_id": cls.country.id,
                "city_id": cls.city.id,
            }
        )

    def test_on_change_city(self):
        self.lead.location_id = self.location.id
        self.lead.on_change_city()
        self.assertEqual(self.lead.zip, "12345")
        self.assertEqual(self.lead.city, "Test city")
        self.assertEqual(self.lead.state_id.name, "Test state")
        self.assertEqual(self.lead.country_id.name, "Test country")

    def test_onchange_partner_id_crm_location(self):
        self.partner.zip_id = self.location.id
        self.lead.partner_id = self.partner.id
        self.lead.onchange_partner_id_crm_location()
        self.assertEqual(self.lead.location_id.name, "12345")


from collections import Counter
def oneOrTwoEditsAway(str1, str2):
	if abs(len(str1) - len(str2)) > 1:
		return False
	elif len(str1) == len(str2):
		cntr = 0
		for s1, s2 in zip(str1, str2):
			if s1 != s2:
				cntr += 1
				if cntr > 1:
					return False
		return True

	short_str = str1
	long_str = str2
	if len(str1) > len(str2):
		short_str = str2
		long_str = str1

	cntr1 = Counter(short_str)
	cntr2 = Counter(long_str)
	unique_char = ""
	unique_counter = 0
	for char in long_str:
		if cntr1[char] != cntr2[char]:
			unique_char = char
			unique_counter += 1
			if unique_counter > 1:
				return False
	if unique_char + short_str == long_str or short_str + unique_char == long_str:
		return True

	for i in range(len(short_str)):
		fixed_str = short_str[0:i+1] + unique_char + short_str[i+1:]
		if fixed_str == long_str:
			return True
	return False

print(oneOrTwoEditsAway("pale", "ple"))
print(oneOrTwoEditsAway("pales", "pale"))
print(oneOrTwoEditsAway("pale", "bale"))
print(oneOrTwoEditsAway("pale", "bake"))
print(oneOrTwoEditsAway("paler", "balre"))
print(oneOrTwoEditsAway("pale", "pake"))


# Your name as it is on the moodle site : Zanqing Feng
# Your email: zafe5975@colorado.edu
# Your student ID: 102077587
# On my honor as a University of Colorado student, I acknowledge that
# I did not receive any unauthorized help for this assignment.
# I understand that systems like MOSS can easily detect code plagiarism.

from __future__ import print_function
import sys

# We will use a class called my trie node
class MyTrieNode:
	# Initialize some fields

	def __init__(self, isRootNode):
		#The initialization below is just a suggestion.
		#Change it as you will.
		# But do not change the signature of the constructor.
		self.isRoot = isRootNode
		self.isWordEnd = False # is this node a word ending node
		self.isRoot = False # is this a root node
		self.count = 0 # frequency count
		self.next = {} # Dictionary mappng each character from a-z to the child node

	def addWord(self,w):
		assert(len(w) > 0)

		# YOUR CODE HERE
		# If you want to create helper/auxiliary functions, please do so.
		if w[0] not in self.next:
			self.next[w[0]] = MyTrieNode(False)
		if len(w) > 1:
			self.next[w[0]].addWord(w[1:])
		else:
			self.next[w[0]].isWordEnd = True
			self.next[w[0]].count += 1

		return

	def lookupWord(self,w):
		# Return frequency of occurrence of the word w in the trie
		# returns a number for the frequency and 0 if the word w does not occur.

		# YOUR CODE HERE
		if len(w) == 0:
			return self.count
		if w[0] not in self.next:
			return 0
		else:
			return self.next[w[0]].lookupWord(w[1:])

	def helper(self, w_completion, w):
		if len(w_completion) < len(w):
			if w[len(w_completion)] in self.next:
				return self.next[w[len(w_completion)]].helper(w_completion + w[len(w_completion)], w)
			else:
				return []
		else:
			res = []
			if self.isWordEnd == True:
				res.append((w_completion, self.count))
			for key in self.next.keys():
				res.extend(self.next[key].helper(w_completion + key, w))
			return res

	def autoComplete(self,w):
		#Returns possible list of autocompletions of the word w
		#Returns a list of pairs (s,j) denoting that
		#         word s occurs with frequency j

		#YOUR CODE HERE
		return self.helper("", w)

if (__name__ == '__main__'):
    t= MyTrieNode(True)
    lst1=['test','testament','testing','ping','pin','pink','pine','pint','testing','pinetree']

    for w in lst1:
        t.addWord(w)

    j = t.lookupWord('testy') # should return 0
    j2 = t.lookupWord('telltale') # should return 0
    j3 = t.lookupWord ('testing') # should return 2
    lst3 = t.autoComplete('pi')
    print('Completions for \"pi\" are : ')
    print(lst3)

    lst4 = t.autoComplete('tes')
    print('Completions for \"tes\" are : ')
    print(lst4)


"""
Brock Francom
A02052161
CS-1400-001
Douglas Galarus
2/15/2018
hw6 - Exercise 5.34
"""

def run():
    #import random and assign variables
    import random
    lotteryDigit1 = 0
    lotteryDigit2 = 0

    #loop is used to generate 2 different digits
    while lotteryDigit1 == lotteryDigit2:
        lottery = random.randint(00, 99)
        lotteryDigit1 = lottery // 10
        lotteryDigit2 = lottery % 10

    #user input
    user = int(input("Enter a number (2 digits): "))

    #evaluate user input
    userDigit1 = user // 10
    userDigit2 = user % 10

    #print results
    if user == lottery:
        print("Exact match! You win $10,000!")
    elif userDigit1 == lotteryDigit2 and userDigit2 == lotteryDigit1:
        print("Matched all digits! You win $3,000!")
    elif userDigit1 == lotteryDigit1 or userDigit1 == lotteryDigit2 \
            or userDigit2 == lotteryDigit1 or userDigit2 == lotteryDigit2:
        print("Matched 1 digit. You win $1,000!")
    else:
        print("Sorry no match.")


from os import path
from tempfile import TemporaryDirectory

from pytest import raises, mark

from duplicate_file_removal.file_record import RecordStatus
from duplicate_file_removal.processor import RecordsProcessor
from tests.utils.util import ignore_exceptions, count, no_stdout

def test_remove_duplicates():
    with raises(RuntimeError):  # make sure paths provided have to be absolute
        RecordsProcessor.remove_duplicates([], ["..\\SomeFolder"])

def test_scanner_results_to_groups(scanner, gen_files_by_specification):
    num_of_dup_files = 10
    num_of_unique_files = 5

    base_dir = gen_files_by_specification(num_of_dup_files, num_of_unique_files)
    res = scanner.scan(base_dir.name)
    res = RecordsProcessor.scanner_results_to_groups(res)
    assert len(res) == 1  # occurrences for unique files filtered, and 1 occurrence for the 10 duplicate files

    dup_files_list = next(iter(res.values()))  # get the value of the first element
    assert len(dup_files_list) == num_of_dup_files

def test_remove_duplicate_normalized(binary_data, scanner):
    """
    Test to insure that given a more specific priority in a less specificity,
    then specificity > priority
    """
    temp_dir_1 = TemporaryDirectory()
    temp_file_name_1 = path.join(temp_dir_1.name, "file_1")
    with open(temp_file_name_1, "wb") as f:
        f.write(binary_data)

    temp_dir_2 = TemporaryDirectory(dir=temp_dir_1.name)
    temp_file_name_2 = path.join(temp_dir_2.name, "file_2")
    with open(temp_file_name_2, "wb") as f:
        f.write(binary_data)

    records = scanner.scan(temp_dir_1.name)
    assert len(records) == 2  # sanity check
    priority = ["C:\\", temp_dir_2.name]

    RecordsProcessor._remove_duplicate_normalized(records, priority)

    assert records[0].status == RecordStatus.deleted
    assert records[1].status == RecordStatus.exists

    # TemporaryDirectory cleanup warning ignored
    ignore_exceptions(temp_dir_1.cleanup)
    ignore_exceptions(temp_dir_2.cleanup)

# noinspection PyTypeChecker
def test_highest_specificity():
    chosen_one = object()

    candidates = [("C:\\", object()), ("C:\\specific", object()), ("C:\\specific\\choose_me", chosen_one)]
    assert chosen_one is RecordsProcessor._highest_specificity(candidates)

@mark.parametrize("is_simulation,expected", [(False, 2), (True, 3)])
def test_delete_records(is_simulation, expected, scanner, gen_files_by_specification):
    # If changed, adjust parametrize value accordingly
    num_of_dup_files = 2
    num_of_unique_files = 5

    base_dir = gen_files_by_specification(num_of_dup_files, num_of_unique_files)
    res = RecordsProcessor.scanner_results_to_groups(scanner.scan(base_dir.name))
    duplicate_records = next(iter(res.values()))

    res.add(duplicate_records[0])  # Try to 'cheat' the scanner and have two FileRecords to the same physical file
    with no_stdout():
        RecordsProcessor.delete_records(duplicate_records[0], duplicate_records, is_simulation)

    assert count(filter(lambda x: x.status == RecordStatus.exists, duplicate_records)) == expected


"""
题目描述:
每年六一儿童节,牛客都会准备一些小礼物去看望孤儿院的小朋友,今年亦是如此。HF作为牛客的资深元老,自然也准备了一些小游戏。
其中,有个游戏是这样的:首先,让小朋友们围成一个大圈。然后,他随机指定一个数m,让编号为0的小朋友开始报数。
每次喊到m-1的那个小朋友要出列唱首歌,然后可以在礼品箱中任意的挑选礼物,并且不再回到圈中,从他的下一个小朋友开始,
继续0...m-1报数....这样下去....直到剩下最后一个小朋友,可以不用表演,并且拿到牛客名贵的“名侦探柯南”典藏版(名额有限哦!!^_^)。
请你试着想下,哪个小朋友会得到这份礼品呢？(注：小朋友的编号是从0到n-1)
如果没有小朋友，请返回-1
解决思路：
第一种：约瑟夫环问题，套用递归公式
第二种：用链表或者数组模拟过程
"""

# -*- coding:utf-8 -*-
# 第一种
# import sys
# sys.setrecursionlimit(10000)  # 设置默认递归次数，系统默认为1000，有些大的数会超出递归次数
# class Solution:
#     def LastRemaining_Solution(self, n, m):
#         if n==0:
#             return -1
#         if n==1:
#             return 0
#         else:
#             return (self.LastRemaining_Solution(n-1, m)+m)%n

# 第二种
class Solution:
    def LastRemaining_Solution(self, n, m):
        if n==0:
            return -1
        if n==1:
            return 0
        numbers = [i for i in range(n)]
        start = 0
        while len(numbers)>1:
            for i in range(1, m):
                start += 1
                if start == len(numbers):
                    start = 0
            numbers.remove(numbers[start])
            if start == len(numbers):
                start = 0
        return numbers.pop()

n, m = 4000, 997
S = Solution()
result = S.LastRemaining_Solution(n, m)
print(result)