import os
import sys
import time
import json
import asyncio
import logging
import re

from opencensus.ext.azure.log_exporter import AzureLogHandler
from azure.iot.device import Message
from azure.iot.device.aio import IoTHubModuleClient
from datetime import datetime, timedelta

TWIN_CALLBACKS = 0
OBJECT_TAGS = ['truck']
OBJECT_CONFIDENCE = 0.5
NOTIFICATION_TIMEOUT = '5m'
EVENT_TIMEOUT_DICT = {}
DATETIME_STRING_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
LOG_LEVEL = 'INFO'

module_client = None
logger = logging.getLogger(__name__)

# Define behavior to keep the application running
async def continuous_loop():
    while True:
        time.sleep(100)

# twin_patch_handler is invoked when the module twin's desired properties are updated
def twin_patch_handler(patch):
    global TWIN_CALLBACKS
    global OBJECT_TAGS
    global OBJECT_CONFIDENCE
    global NOTIFICATION_TIMEOUT
    global LOG_LEVEL

    logger.debug('The data in the desired properties patch was: %s' % patch)

    if 'objectTags' in patch:
        OBJECT_TAGS = patch['objectTags']
    if 'objectConfidence' in patch:
        OBJECT_CONFIDENCE = patch['objectConfidence']
    if 'notificationTimeout' in patch:
        NOTIFICATION_TIMEOUT = patch['notificationTimeout']
    if 'logLevel' in patch:
        LOG_LEVEL = patch['logLevel']
        logger.setLevel(LOG_LEVEL)

    TWIN_CALLBACKS += 1
    logger.debug('Total calls confirmed: %d\n' % TWIN_CALLBACKS)

# Define behavior for receiving an input message on the 'detectedObjects' route
async def message_handler(message):
    if message.input_name == 'detectedObjects':
        logger.debug('Message received on detectedObjects route')
        await object_detected_handler(message)
    else:
        logger.warning('Message received on unknown input route')

# object_detected_handler checks the AI inferrencing result and sends a message to the
# 'eventAlertTrigger` output message route when all conditions are met
async def object_detected_handler(input_message):
    try:
        if input_message is None:
            logger.warning('input_message is None')
            return

        # List to hold inferences with desired tag and confidence level
        inferences = []

        message = input_message.data.decode('utf-8')
        data = json.loads(message)

        detected_objects = data['inferences']
        graph_instance = extract_graph_instance_from_subject(input_message.custom_properties['subject'])

        logger.info(f'{graph_instance}: Detected {len(detected_objects)} object(s)')

        for inference in detected_objects:
            entity = inference['entity']
            tag = entity['tag']
            confidence = tag['confidence']
            detected_object = tag['value']

            event_data = {
                'detected_object': detected_object,
                'confidence': confidence,
                'graph_instance': graph_instance
            }

            logger.info(f'{graph_instance}: {detected_object} detected. Confidence: {confidence}', extra={'custom_dimensions': event_data})

            if (detected_object.lower() in OBJECT_TAGS) and (confidence > OBJECT_CONFIDENCE):
                logger.info(f'>>>>> Match found: {tag} <<<<<')
                inferences.append(inference)

        if len(inferences) > 0:
            event_time = input_message.custom_properties['eventTime']
            should_send_message = check_event_timeout(event_time, input_message.custom_properties['subject'])

            logger.debug(f'Send message: {should_send_message}')

            if should_send_message:
                output_message = construct_output_message(
                    graph_instance,
                    input_message.custom_properties['$.cdid'],
                    inferences,
                    event_time)
                output_message_route = 'Event-' + graph_instance
                await module_client.send_message_to_output(output_message, output_message_route)

    except Exception as ex:
        logger.exception('Unexpected error: %s' % ex)

# extract_graph_instance_from_subject extracts the graph instance name from the subject
# e.g. given "/graphInstances/Truck/processors/inferenceClient" the output will be "Truck"
def extract_graph_instance_from_subject(subject):
    if subject is None:
        raise Exception('subject is None')

    graph_instance_signature = '/graphInstances/'

    if graph_instance_signature not in subject:
        raise Exception('Graph instance signature is not present in subject')

    words = subject.split('/')
    return words[2]

# check_event_timeout determines if we should send a message for a given graph instance based on the
# notification time out value. For example, we may only want to send a message for the same graph instance
# every 5 minutes to avoid spam messages.
def check_event_timeout(event_time, subject):
    graph_instance = extract_graph_instance_from_subject(subject)
    input_msg_event_time_object = datetime.strptime((event_time), DATETIME_STRING_FORMAT)

    if NOTIFICATION_TIMEOUT == '0' or NOTIFICATION_TIMEOUT == '0s':
        EVENT_TIMEOUT_DICT[graph_instance] = 0
        return True

    logger.debug(f'Current input time: {input_msg_event_time_object}')

    if graph_instance in EVENT_TIMEOUT_DICT and input_msg_event_time_object <= EVENT_TIMEOUT_DICT[graph_instance]:
        logger.debug(f'This message already exists, but it\'s timeout of {NOTIFICATION_TIMEOUT} is still active')
        return False
    else:
        timeout_seconds = time_to_seconds(NOTIFICATION_TIMEOUT)

        if timeout_seconds == -1:
            timeout_seconds = 900
            logger.warning('NOTIFICATION_TIMEOUT is not configured correctly, defaulting to 15m')

        EVENT_TIMEOUT_DICT[graph_instance] = input_msg_event_time_object + timedelta(seconds=timeout_seconds)

        logger.debug('Adding this event for the first time or updating to a new timeout')
        logger.debug(f'New input time: {EVENT_TIMEOUT_DICT[graph_instance]}')

        return True

# construct_output_message creates the output message we will send to IoT Hub
def construct_output_message(graph_instance, device_id, inferences, event_time):
    logger.debug("Constructing output message")

    output_dict = {
        'graphInstance': graph_instance,
        'deviceId': device_id,
        'inferences': inferences,
        'eventTime': event_time
    }

    output_message_string = json.dumps(output_dict)
    output_message = Message(output_message_string, content_encoding='utf-8')

    logger.debug(f'Output message: {output_message}')
    return output_message

# Function which translates time from format '(\d+)([smh]' to value in seconds.
def time_to_seconds(time_string):
    if time_string is None:
        logger.error('Input string cannot be None')
        return -1

    time_re = re.search(r'(\d+)([smh])', time_string)

    if time_re is None or len(time_re.groups()) < 2:
        logger.error('Could not parse time')
        return -1

    time_unit = time_re.groups()[1]
    time_value = int(time_re.groups()[0])

    if time_unit == 'm':
        time_value *= 60
    elif time_unit == 'h':
        time_value *= 3600
    return time_value

async def main():
    global module_client

    logger.setLevel(LOG_LEVEL)

    # ODBL: Object Detection Business Logic
    formatter = logging.Formatter('[ODBL] [%(asctime)-15s] [%(threadName)-12.12s] [%(levelname)s]: %(message)s')

    # Add stdout handler
    stdout_handler = logging.StreamHandler(sys.stdout)
    stdout_handler.setFormatter(formatter)
    logger.addHandler(stdout_handler)

    appinsights_instrumentationkey = os.environ.get('APPINSIGHTS_INSTRUMENTATIONKEY', None)
    if appinsights_instrumentationkey and not appinsights_instrumentationkey.isspace():
        try:
            azure_log_handler = AzureLogHandler(instrumentation_key=appinsights_instrumentationkey)
            azure_log_handler.setFormatter(formatter)
            logger.addHandler(azure_log_handler)

            logger.info('Application Insights initialized.')
        except Exception:
            logger.exception('Application Insights failed to initialize')

    try:
        if not sys.version >= '3.6':
            raise Exception('The object detection business logic module requires python 3.6+. Current version of Python: %s' % sys.version)

        logger.info('Starting the object detection business logic module ...')

        module_client = IoTHubModuleClient.create_from_edge_environment(websockets=True)

        await module_client.connect()
        module_client.on_twin_desired_properties_patch_received = twin_patch_handler
        module_client.on_message_received = message_handler

        logger.info('The object detection business logic module is now waiting for messages.')

        await continuous_loop()
        await module_client.disconnect()
    except Exception as ex:
        logger.exception('Unexpected error: %s' % ex)
        return

if __name__ == '__main__':
    asyncio.run(main())


import os
import numpy as np
import pandas as pd
import time

def measure_elapsed_time(function):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = function(*args, **kwargs)
        end_time = time.time()
        elapsed_time = end_time - start_time
        print(f"Elapsed time: {elapsed_time:.3f} seconds")
        return result
    return wrapper

@measure_elapsed_time
def batch_data_augmentation(input_dir: str):
    joint_pairs = [('right_shoulder', 'right_elbow'), ('right_elbow', 'right_wrist'), ('right_hip', 'right_knee'), ('right_knee', 'right_ankle'), ('right_ankle', 'right_foot_index'), ('left_shoulder', 'left_elbow'), ('left_elbow', 'left_wrist'), ('left_hip', 'left_knee'), ('left_knee', 'left_ankle'), ('left_ankle', 'left_foot_index'), ('forehead', 'torso')]

    def calculate_additional_columns(kd):
        kd.loc[:, 'forehead_x'] = (kd['left_eye_outer_x'] + kd['right_eye_outer_x']) / 2
        kd.loc[:, 'forehead_y'] = (kd['left_eye_outer_y'] + kd['right_eye_outer_y']) / 2
        kd.loc[:, 'forehead_z'] = (kd['left_eye_outer_z'] + kd['right_eye_outer_z']) / 2

        kd.loc[:, 'torso_x'] = (kd['left_shoulder_x'] + kd['right_shoulder_x'] + kd['left_hip_x'] + kd['right_hip_x']) / 4
        kd.loc[:, 'torso_y'] = (kd['left_shoulder_y'] + kd['right_shoulder_y'] + kd['left_hip_y'] + kd['right_hip_y']) / 4
        kd.loc[:, 'torso_z'] = (kd['left_shoulder_z'] + kd['right_shoulder_z'] + kd['left_hip_z'] + kd['right_hip_z']) / 4

        for joint in joint_pairs:
            angles = []
            distances = []
            velocities = []
            for i in range(len(kd)):
                joint1 = np.array([kd[f"{joint[0]}_x"].iloc[i], kd[f"{joint[0]}_y"].iloc[i], kd[f"{joint[0]}_z"].iloc[i]])
                joint2 = np.array([kd[f"{joint[1]}_x"].iloc[i], kd[f"{joint[1]}_y"].iloc[i], kd[f"{joint[1]}_z"].iloc[i]])
                dot_product = np.dot(joint1, joint2)
                magnitude1 = np.linalg.norm(joint1)
                magnitude2 = np.linalg.norm(joint2)
                distance = np.linalg.norm(joint1 - joint2)
                angle = np.degrees(np.arccos(dot_product / (magnitude1 * magnitude2)))
                angles.append(angle)
                distances.append(distance)

                if i == 0:
                    velocity = 0
                    velocities.append(velocity)
                else:
                    velocity = (distances[i] - distances[i-1]) * fps
                    velocities.append(velocity)

            kd[f"a_{joint[0]}_{joint[1]}"] = angles
            kd[f"d_{joint[0]}_{joint[1]}"] = distances
            kd[f"v_{joint[0]}_{joint[1]}"] = velocities

    for filename in os.listdir(input_dir):
        if filename.endswith('.csv'):
            kd = pd.read_csv(f'{input_dir}/{filename}')
            fps = int(kd['fps'][0])
            kd = kd[::fps].reset_index(drop=True)
            calculate_additional_columns(kd)
            filename_without_ext = filename.split('.')[0]
            kd.to_csv(f'./data/processed/{filename_without_ext}.csv')
            print(f'{filename_without_ext} has been processed!')


import numpy as np
import math
import random
import sys
import itertools
from pathlib import Path
home = str(Path.home())
import sys
rankability_path = "%s/rankability_toolbox_dev"%home
if rankability_path not in sys.path:
    sys.path.insert(0,rankability_path)
import pyrankability
from pyrankability.rank import solve
from pyrankability.search import solve_pair_min_tau
import json
from tqdm import tqdm
from scipy import stats
from utilities import *

######## STATISTICS OF P ########
# Functions of P which package certain similar statistics
# together for sake of efficiency. These functions will probably
# replace the current "RankabilityMetric" classes since running
# cross-validated linear models seems to be the better alternative
# to hand-crafting metrics.

# This function found at: https://stackoverflow.com/a/48916127
# TODO: Check to make sure you dont need np.argsort
def kendall_w(expt_ratings):
    expt_ratings = np.array(expt_ratings)
    if expt_ratings.ndim!=2:
        raise 'ratings matrix must be 2-dimensional'
    m = expt_ratings.shape[0] # number of raters
    n = expt_ratings.shape[1] # number of items
    denom = m**2*(n**3-n)
    rating_sums = np.sum(expt_ratings, axis=0)
    S = n*np.var(rating_sums)
    return "kendall_w", 12*S/denom

# "Lowerbound" because many random restarts may not find full P
# TODO: Add SCIP compatibility to grab full P set rather than estimate
def p_len(P):
    return "p_lowerbound", len(P)

def l2_dist_stats(P):
    p = len(P)
    max_dist = 0.0
    mean_dist = 0.0
    for r1 in range(p):
        for r2 in range(r1+1, p):
            dist = np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))
            mean_dist += dist
            if dist > max_dist:
                max_dist = dist

    if p > 1:
        mean_dist /= (p*(p-1)/2.0)

    return ["max_L2_dist", "mean_L2_dist"], [max_dist, mean_dist]

def tau_stats(P):
    p = len(P)
    min_tau = 1.0
    mean_tau = 0.0
    for r1 in range(p):
        for r2 in range(r1+1, p):
            tau = kendall_tau(P[r1], P[r2])
            mean_tau += tau
            if tau < min_tau:
                min_tau = tau

    if p > 1:
        mean_tau /= (p*(p-1)/2.0)
    else:
        mean_tau = 1.0

    return ["min_tau", "mean_tau"], [min_tau, mean_tau]

def get_P_stats(P):
    # Ensure P is a set, then make addressable as a list
    P = list(set(P))
    results = {}
    for func in [kendall_w, p_len, l2_dist_stats, tau_stats]:
        stat_names, stat_values = func(P)
        if type(stat_names) == list:
            for i in range(len(stat_names)):
                results[stat_names[i]] = stat_values[i]
        else:
            results[stat_names] = stat_values
    return results

######## RANKABILITY METRICS ########
# Not currently employed by the core testbed

class RankabilityMetric:

    def compute(self, k, details):
        P = list(set(details["P"]))
        return self._compute(k, P)

    def _compute(self, k, P):
        # Child classes should compute their rankability metric from k and P
        raise NotImplemented("Don't use the generic RankabilityMetric class")

class RatioToMaxMetric(RankabilityMetric):
    def _compute(self, k, P):
        n = len(P[0])
        return 1.0 - (k*len(P) / ((n**2 - n)/2 * math.factorial(n)))

class KendallWMetric(RankabilityMetric):

    def _compute(self, k, P):
        n = len(P[0])
        _, w = kendall_w(P)
        return 1 - ((k / (n**3 - n**2)) * (1 - w))

class L2DifferenceMetric(RankabilityMetric):

    def __init__(self, strategy="max"):
        strategy=strategy.lower()
        self.strategy = strategy
        if strategy == "max":
            self.get_dist_stat = self.get_max_dist
        elif strategy == "mean":
            self.get_dist_stat = self.get_mean_dist
        else:
            raise ValueError("Unrecognized L2DifferenceMetric strategy: %s" % strategy)

    def get_mean_dist(self, P):
        p = len(P)
        mean_dist = 0.0
        for r1 in range(p):
            for r2 in range(r1+1, p):
                mean_dist += np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))

        if p == 1:
            return mean_dist
        else:
            return mean_dist / (p*(p-1)/2.0)

    def get_max_dist(self, P):
        p = len(P)
        max_dist = 0.0
        for r1 in range(p):
            for r2 in range(r1+1, p):
                dist = np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))
                if dist > max_dist:
                    max_dist = dist
        return max_dist

    def _compute(self, k, P):
        p = len(P)
        n = len(P[0])
        dist = self.get_dist_stat(P)
        return 1.0 - (k / (n**3 - n**2) * (dist / np.sqrt((n / 3) * (n**2 - 1))))

class MeanTauMetric(RankabilityMetric):
    # Two similar statistics exist for the use of mean tau
    #      "hays"  -->  W_a defined by Hays (1960)
    # "ehrenberg"  -->  W_t defined by Ehrenberg (1952)
    #
    # The two strategies are the same when m (# of rankings) is even
    # Hays' W_a is computed with m+1 instead of m when m is odd
    # This ensures that 0 is the minimum value at all values of m
    #
    # In keeping with the statistics literature I've read:
    #    m  <-- number of rankings (equivalent to p)
    #    n  <-- number of items to rank / length of rankings
    #    u  <-- mean kendall tau over all pairs of rankings

    def __init__(self, strategy="hays"):
        strategy = strategy.lower()
        self.strategy = strategy
        if strategy == "hays":
            self.get_W = self.get_W_a
        elif strategy == "ehrenberg":
            self.get_W = self.get_W_t
        else:
            raise ValueError("Unrecognized MeanTauMetric strategy: %s" % strategy)

    # Defined by Ehrenberg (1952) as a possible measure of concordance
    def get_W_t(self, m, u):
        return ((m - 1.0) * u + 1.0) / m

    # Defined by Hays (1960) so that a minimum of 0 is always possible for all values of m
    def get_W_a(self, m, u):
        if m & 1 == 1:
            m += 1
        return ((m - 1.0) * u + 1.0) / m

    def _compute(self, k, P):
        p = len(P)
        if p == 1:
            return 1.0
        if p == 2:
            u = kendall_tau(P[0], P[1])
        else:
            u = 0.0
            for r1 in range(p):
                for r2 in range(r1+1, p):
                    tau = kendall_tau(P[r1], P[r2])
                    u += tau
            u /= p*(p-1)/2
        n = len(P[0])
        w_stat = self.get_W(p, u)
        return 1 - (k / (n**3 - n**2) * (1.0 - w_stat))

######## NOISE GENERATORS ########

class NoiseGenerator:

    def apply_noise(self, D):
        # Child classes should return numpy array of D_tilde
        raise NotImplemented("Don't use the generic NoiseGenerator class")

class IdentityNoise(NoiseGenerator):
    def apply_noise(self, D):
        return D

    def __str__(self):
        return "IdentityNoise()"

class BinaryFlipNoise(NoiseGenerator):
    def __init__(self, noisePercentage):
        self.noisePercentage = noisePercentage

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_flips = (np.square(n) - n) * self.noisePercentage
        unique_elems = set()
        for flip in range(int(num_flips)):
            i, j = random.sample(range(n), 2) #Ensures that i and j are distinct
            while ((i, j) in unique_elems): i, j = random.sample(range(n), 2)
            unique_elems.add((i, j))
            D_noisy[i][j] = 1 - D_noisy[i][j]
        return D_noisy

    def __str__(self):
        return "BinaryFlipNoise({})".format(self.noisePercentage)

class SwapNoise(NoiseGenerator):
    # Flips (swaps edge weights for (i,j) and (j,i)) for given percentage
    # of relationships. 100% noise corresponds to returning D.T

    def __init__(self, noisePercentage):
        self.noisePercentage = noisePercentage

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_swaps = int((n*n - n)/2 * self.noisePercentage)
        if self.noisePercentage > 0.5:
            D_noisy = D_noisy.T
            num_swaps = int((n*n - n)/2 - num_swaps)
        # Get indices of upper-triangular elements
        i_arr, j_arr = np.triu_indices(n,1)
        num_offdiag = len(i_arr)
        indices = random.sample(range(num_offdiag), num_swaps)
        for idx in indices:
            i, j = i_arr[idx], j_arr[idx]
            # Reverse this relationship
            temp = D_noisy[i,j]
            D_noisy[i][j] = D_noisy[j][i]
            D_noisy[j][i] = temp
        return D_noisy

    def __str__(self):
        return "SwapNoise({})".format(self.noisePercentage)

class BootstrapResamplingNoise(NoiseGenerator):

    def __init__(self, noisePercentage, byRow=True):
        self.noisePercentage = noisePercentage
        self.byRow = byRow

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_resampled = int((n*n - n) * self.noisePercentage)
        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))

        # Select the elements to be resampled and the bootstrap samples to use
        num_offdiag = len(i_arr)
        indices = random.sample(range(num_offdiag), num_resampled) # Without replacement
        if self.byRow:
            # Replaces elements from ones in the same row (resampling only within the row)
            bootstrap = np.random.randint(n-1, size=num_resampled) # With replacement
            for idx, rel_idx in enumerate(indices):
                # rel_idx is the index of the off-diagonal relationship to be replaced
                # idx is the index of the bootstrap sample to replace it with from the same row
                bs_idx = bootstrap[idx]
                # Replace the chosen off-diag element of D_noisy using the
                # off-diag element chosen from the same row
                row = i_arr[rel_idx]
                old_col = j_arr[rel_idx]
                new_col = bs_idx if bs_idx < row else bs_idx + 1
                D_noisy[row, old_col] = D[row, new_col]
        else:
            # Replaces elements with random off-diagonal element from entire D
            bootstrap = np.random.randint(num_offdiag, size=num_resampled) # With replacement
            for idx, rel_idx in enumerate(indices):
                # rel_idx is the index of the off-diagonal relationship to be replaced
                # idx is the index of the bootstrap sample to replace it with
                bs_rel_idx = bootstrap[idx]
                # Replace the chosen off-diag element of D_noisy using the
                # off-diag element chosen from D
                D_noisy[i_arr[rel_idx], j_arr[rel_idx]] = D[i_arr[bs_rel_idx], j_arr[bs_rel_idx]]

        return D_noisy

    def __str__(self):
        return "BootstrapResamplingNoise({},{})".format(self.noisePercentage, self.byRow)

class NormalResamplingNoise(NoiseGenerator):

    def __init__(self, noisePercentage, clip_range=False):
        self.noisePercentage = noisePercentage
        self.clip_range = clip_range

    # TODO: Change from replacement to additive noise
    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_resampled = int((n*n - n) * self.noisePercentage)

        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))
        offdiags = D[i_arr, j_arr]

        # Get the range of the off-diagonal elements if clipping
        if self.clip_range:
            D_min = np.amin(offdiags)
            D_max = np.amax(offdiags)

        # Fit a normal distribution to all off-diagonal elements
        mu, sigma = stats.norm.fit(offdiags)

        # Select the elements to be resampled
        indices = random.sample(range(len(i_arr)), num_resampled) # Without replacement

        for rel_idx in indices:
            # rel_idx is the index of the off-diagonal relationship to be replaced
            # Replace the chosen off-diag element of D_noisy using a
            # sample from the normal distribution fit to the data
            sample = stats.norm.rvs(loc=mu, scale=sigma)
            if self.clip_range:
                sample = np.clip(sample, D_min, D_max)
            D_noisy[i_arr[rel_idx], j_arr[rel_idx]] = sample

        return D_noisy

    def __str__(self):
        return "NormalResamplingNoise({})".format(self.noisePercentage)

class NormalAdditiveNoise(NoiseGenerator):

    def __init__(self, noisePercentage, varPercentage=1.0, explicitVar=None):
        self.noisePercentage = noisePercentage
        self.varPercentage = varPercentage
        self.explicitVar = explicitVar

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_noised = int((n*n - n) * self.noisePercentage)

        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))
        offdiags = D[i_arr, j_arr]

        # Calculate variance of additive noise
        if self.explicitVar is not None:
            sigma = self.explicitVar * self.varPercentage
        else:
            _, sigma = stats.norm.fit(offdiags)
            sigma = sigma * self.varPercentage

        # Select the elements to be noised
        indices = random.sample(range(len(i_arr)), num_noised) # Without replacement

        for rel_idx in indices:
            D_noisy[i_arr[rel_idx], j_arr[rel_idx]] += stats.norm.rvs(loc=0, scale=sigma)

        return D_noisy

    def __str__(self):
        return "NormalAdditiveNoise({},{})".format(self.noisePercentage, self.varPercentage)

######## DATA SOURCES ########

class DataSource:

    def init_D(self):
        # Child classes should return numpy array
        raise NotImplemented("Don't use the generic DataSource class")

class LOLib(DataSource):
    def __init__(self, file_name, full_path=False):
        self.n = 0
        self.file_name = file_name
        self.full_path = full_path

    def get_filepath(self):
        if self.full_path:
            return self.file_name
        else:
            return "./lolib_data/" + self.file_name

    def get_n(self):
        if self.n is None or self.n < 1:
            with open(self.get_filepath(), 'r') as f:
                self.n = int(f.readline())
        return self.n

    def init_D(self):
        with open(self.get_filepath(), 'r') as f:
            unparsed = f.read().splitlines()
        self.n = int(unparsed[0]) #0th element is dim of D
        elements = [row.split(' ') for row in unparsed[1:]]
        D = []
        for row in elements:
            D.append([int(e) for e in row if e != ''])
        return np.array(D)

    def __str__(self):
        return "LOLib({})".format(self.file_name)

class PerfectBinarySource(DataSource):

    def __init__(self, n):
        self.n = n

    def init_D(self):
        D = np.zeros((self.n,self.n), dtype=int)
        D[np.triu_indices(self.n,1)] = 1
        return D

    def __str__(self):
        return "PerfectBinarySource({})".format(self.n)

class PerfectWeightedSource(DataSource):

    def __init__(self, n, scale=1):
        self.n = n
        self.scale = scale

    def init_D(self):
        D = np.zeros((self.n,self.n), dtype=float)
        i_arr, j_arr = np.triu_indices(self.n,1)
        for idx in range(len(i_arr)):
            i, j = i_arr[idx], j_arr[idx]
            D[i, j] = self.scale * (-i+j)
        return D

    def __str__(self):
        return "PerfectWeightedSource({},{})".format(self.n, self.scale)

class SynthELOTournamentSource(DataSource):
    # Simulates a tournament played by competitors with
    # normally distributed ELO scores
    # This ELO system uses a logistic curve to predict win probability

    def __init__(self, n, n_games=5, comp_var=200, elo_scale=400):
        # n --> the number of competitors
        # n_games --> the number of games played between every pair of competitors
        # comp_var --> the variance of the normal distribution competitor ELOs are drawn from
        #     a smaller comp_var/elo_scale ratio will produce a very rankable matrix
        #     a larger comp_var/elo_scale ratio will produce a very unrankable matrix
        # elo_scale --> the scale parameter for the ELO system
        self.n = n
        self.n_games = n_games
        self.comp_var = comp_var
        self.elo_scale = elo_scale

    def sigmoid(self, x):
        return 1 / (1 + math.exp(-x))

    def init_D(self):
        ELOs = sorted(stats.norm.rvs(scale=self.comp_var, size=self.n), reverse=True)
        D = np.zeros((self.n,self.n), dtype=float)
        i_arr, j_arr = np.triu_indices(self.n,1)
        for idx in range(len(i_arr)):
            i, j = i_arr[idx], j_arr[idx]
            scaled_elo_diff = (ELOs[i] - ELOs[j])/self.elo_scale
            prob_i_win = self.sigmoid(scaled_elo_diff)
            i_wins = np.random.binomial(n=self.n_games, p=prob_i_win)
            j_wins = self.n_games - i_wins
            D[i, j] = i_wins
            D[j, i] = j_wins
        return D

    def __str__(self):
        return "SynthELOTournamentSource({},{},{},{})".format(self.n, self.n_games, self.comp_var, self.elo_scale)

class UniformRandomSource(DataSource):
    # Every off-diagonal element of the matrix is drawn independently from a uniform
    # distribution over the range of integers [0, max_val].

    def __init__(self, n, max_val=1):
        # n --> the number of items
        # max_val --> the maximum value that can appear as an element in the matrix
        self.n = n
        self.max_val = max_val

    def init_D(self):
        D = np.random.randint(low=0, high=self.max_val+1, size=(self.n,self.n), dtype=int)
        for i in range(self.n):
            D[i,i] = 0
        return D

    def __str__(self):
        return "UniformRandomSource({},{})".format(self.n, self.max_val)

######## RANKING ALGORITHMS ########

class RankingAlgorithm:

    def rank(D):
        # Child classes should return numpy array of ranking vector
        raise NotImplemented("Don't use the generic RankingAlgorithm class")

class LOPRankingAlgorithm(RankingAlgorithm):

    def rank(self, D):
        k, details = solve(D)
        # This could return the full P set or randomly sample from it rather
        # than reporting the first it finds.
        return details["P"][0]

    def __str__(self):
        return "LOPRankingAlgorithm"

class ColleyRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        b = [1 + (wins[i] - losses[i])/2 for i in range(0,D.shape[0])]
        C = np.zeros(D.shape)
        for i in range(0,D.shape[0]):
            C[i][i] = 2 + totalevents[i]
        for x in range(D.shape[0]):
            for y in range(D.shape[1]):
                if x != y:
                    C[x][y] = (D[x][y] + D[y][x]) * -1
        r = np.linalg.solve(C, b)
        r = sorted([(r[i - 1], i) for i in range(1, D.shape[0] + 1)])
        retvec = [r[i][1]-1 for i in range(len(r)-1, -1, -1)]
        return retvec

    def __str__(self):
        return "ColleyRankingAlgorithm"

class MasseyRankingAlgorithm(RankingAlgorithm):

    def rank(self, D):
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        b = [1 + (wins[i] - losses[i])/2 for i in range(0,D.shape[0])]
        C = np.zeros(D.shape)
        for i in range(0,D.shape[0]):
            C[i][i] = totalevents[i]
        for x in range(D.shape[0]):
            for y in range(D.shape[1]):
                if x != y:
                    C[x][y] = (D[x][y] + D[y][x]) * -1
        C[D.shape[0] - 1] = np.ones(D.shape[0])
        b[D.shape[0] - 1] = 0
        r = np.linalg.solve(C, b)
        r = sorted([(r[i - 1], i) for i in range(1, D.shape[0] + 1)])
        retvec = [r[i][1]-1 for i in range(len(r)-1, -1, -1)]
        return retvec

    def __str__(self):
        return "MasseyRankingAlgorithm"

class MarkovChainRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        #f = np.vectorize(lambda x: 1 if x > 0 else 0)
        '''for i in range(D.shape[0]):
            for j in range(i, D.shape[1]):
                if D[i][j] > D[j][i] and i != j:
                    D[i][j] = 1
                    D[j][i] = 0
                elif D[i][j] < D[j][i] and i != j:
                    D[j][i] = 1
                    D[i][j] = 0
        '''
        V = np.transpose(D.astype(float))
        n = D.shape[0]
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        #print("totalevents: ", totalevents)
        maxevents = max(totalevents)
        #print(V)
        for i in range(V.shape[0]):
            if sum(V[i]) != 0:
                V[i] = np.divide(V[i], sum(V[i]))
            else:
                V[i] = np.zeros(n)
                V[i][i] = 1
        print(V)
        eigenvals, eigenvecs = np.linalg.eig(V)
        #print("eigenvals:\n", eigenvals)
        #for i in range(len(eigenvecs)):
        #    print(eigenvecs[i])
        #print("eigenvecs:\n", eigenvecs)
        print("Max eigenvalue/vector:", max(eigenvals), eigenvecs[np.argmax(eigenvals)]/np.linalg.norm(eigenvecs[np.argmax(eigenvals)]))
        if True in np.iscomplex(eigenvecs[np.argmax(eigenvals)]):
            print("Complex rating vector")
            return
        return np.argsort(eigenvecs[np.argmax(eigenvals)])

    def __str__(self):
        return "MarkovChainRankingAlgorithm"

class MarkovModifiedRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        V = D.astype(float)
        n = D.shape[0]
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        maxevents = max(totalevents)
        #print(V)
        for i in range(V.shape[0]):
            if sum(V[i]) != 0:
                V[i] = np.divide(V[i], sum(V[i]))
            else:
                V[i] = np.zeros(n)
                V[i][i] = 1
        #print(V)
        return np.argsort(V.sum(axis=0))

    def __str__(self):
        return "MarkovModifiedRankingAlgorithm"

####### SOME HELPFUL GLOBALS #######
# These are the defaults used by the testbed

ALL_RANKING_ALGS = [LOPRankingAlgorithm(), MasseyRankingAlgorithm(), ColleyRankingAlgorithm()]
LOW_INTENSITY_NOISE_GENS = [SwapNoise(0.05), BinaryFlipNoise(0.05)]
CORRELATION_METRICS = [kendall_tau, stats.spearmanr]

######## PROBLEM INSTANCE ########

class ProblemInstance:

    def __init__(self, dataSource):
        self.dataSource = dataSource
        self._D = None

    def get_D(self, refresh=False):
        if self._D is None or refresh:
            self._D = self.dataSource.init_D()
        return self._D

    def get_optimal_rankings(self,
                             model="lop",
                             num_random_restarts=0):
        if model in ["lop", "hillside"]:
             return solve(self.get_D(), method=model, num_random_restarts=num_random_restarts, lazy=True)
        else:
            raise ValueError("Unrecognized model name '{}' should be one of 'lop' or 'hillside'".format(model))

    def get_most_distant_optimal_rankings(self,
                                          model="lop"):
        if model in ["lop","hillside"]:
            return solve_pair_min_tau(self.get_D(), method=model, verbose=False)
        else:
            raise ValueError("Unrecognized model name '{}' should be one of 'lop' or 'hillside'".format(model))

    def collect_data(self,
                     ranking_algorithms=ALL_RANKING_ALGS,
                     noise_generators=LOW_INTENSITY_NOISE_GENS,
                     correlation_metrics=CORRELATION_METRICS,
                     model="lop",
                     num_random_restarts=200,
                     n_sensitivity_trials=50):
        D = self.get_D(refresh=True)
        total_weight = np.sum(D)
        k, details = self.get_optimal_rankings(model=model,
                                               num_random_restarts=num_random_restarts)
        P = details["P"]
        k_furthest, details_furthest = self.get_most_distant_optimal_rankings(model=model)
        if details_furthest is not None:
            P.append(details_furthest["perm_x"])
            P.append(details_furthest["perm_y"])
        data = get_P_stats(P)
        data["k"] = k
        if model == "lop":
            data["degree_of_linearity"] = k / total_weight
        else:
            k_lop, _ = self.get_optimal_rankings(model="lop", num_random_restarts=0)
            data["degree_of_linearity"] = k_lop / total_weight
        data["model"] = model
        data["D"] = json.dumps(D.tolist())
        data["Source"] = str(self.dataSource)
        data["n_items"] = D.shape[0]
        data["P"] = str(list(set(P)))
        data["P_repeats"] = str(P)

        for rankingAlg in ranking_algorithms:
            perfect_ranking = rankingAlg.rank(D)
            data["{}_DOL".format(str(rankingAlg))] = np.triu(D[:,perfect_ranking][perfect_ranking,:],1).sum() / total_weight
            for noiseGenerator in noise_generators:
                # Measure the similarity between original ranking and post-noise ranking
                # for many samples of noise.
                taus = []
                rhos = []
                for trial_index in range(n_sensitivity_trials):
                    D_noisy = noiseGenerator.apply_noise(D)
                    noisy_ranking = rankingAlg.rank(D_noisy)
                    tau = kendall_tau(perfect_ranking, noisy_ranking)
                    taus.append(tau)
                    rho = spearman_r(perfect_ranking, noisy_ranking)
                    rhos.append(rho)

                # Transforms taus into "sensitivity scores" such that higher values mean more sensitive to noise
                sensitivities = (1.0 - np.array(taus)) / 2.0
                mean_tau_name = "mean_tau_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[mean_tau_name] = np.mean(sensitivities)
                std_tau_name = "std_tau_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[std_tau_name] = np.std(sensitivities)

                sensitivitiesrho = (1.0 - np.array(rhos)) / 2.0
                mean_rho_name = "mean_rho_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[mean_rho_name] = np.mean(sensitivitiesrho)
                std_rho_name = "std_rho_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[std_rho_name] = np.std(sensitivitiesrho)

        return data

######## MAIN, FOR DEBUG ONLY ########

def main():
    mcra = MarkovChainRankingAlgorithm()
    mmra = MarkovModifiedRankingAlgorithm()

    testmatrix = PerfectBinarySource(10)
    perf = testmatrix.init_D()

    eloTournament = SynthELOTournamentSource(10, 5, 260, 800)
    eloMatrix = eloTournament.init_D()
    eloTournament5 = SynthELOTournamentSource(5, 5, 260, 800)
    eloMatrix5 = eloTournament5.init_D()
    eloTournament2 = SynthELOTournamentSource(3, 5, 260, 800)
    eloMatrix2 = eloTournament2.init_D()
    eloTournament4 = SynthELOTournamentSource(4, 5, 260, 800)
    eloMatrix4 = eloTournament4.init_D()

    fivegood = np.array([[0,1,1,1,0],
                        [0,0,1,1,1],
                        [0,1,0,1,1],
                        [0,0,0,0,1],
                        [3,3,3,3,0]])

    np.set_printoptions(formatter={'float': lambda x: str(x)})

    print(eloMatrix5)
    print(mmra.rank(eloMatrix5))
    print(eloMatrix)
    print(mmra.rank(eloMatrix))
    print(eloMatrix2)
    print(mmra.rank(eloMatrix2))
    print(eloMatrix4)
    print(mmra.rank(eloMatrix4))

#if __name__ == "__main__":
#    main()


while True:
    print("Options")
    print("1.Addition")
    print("2.Subtraction")
    print("3.Division")
    print("4.Multiply")
    commandtxt=input("Choose command: ")
    num1=float(input("First Number: "))
    num2=float(input("Second Number: "))
    if(commandtxt=="Addition"):
        rslt=num1+num2
        print(rslt)
        cnt=input("Do you want to continue? (Yes/No)")
        if(cnt=="No"):
            break
        else:
            continue
    elif(commandtxt=="Subtraction"):
        rslt=num1-num2
        print(rslt)
        cnt=input("Do you want to continue? (Yes/No)")
        if(cnt=="No"):
            break
        else:
            continue
    elif(commandtxt=="Division"):
        try:
            rslt=num1/num2
            print(rslt)
            cnt=input("Do you want to continue? (Yes/No)")
            if(cnt=="No"):
                break
            else:
                continue
        except:
            print("Num2 cannot be equal to 0!")
            cnt=input("Do you want to continue? (Yes/No)")
            if(cnt=="No"):
                break
            else:
                continue

    elif(commandtxt=="Multiply"):
        rslt=num1*num2
        print(rslt)
        cnt=input("Do you want to continue? (Yes/No)")
        if(cnt=="No"):
            break
        else:
            continue
    else:
        print("Wrong Command!")
        cnt2=input("Do you want to try again? (Yes/No)")
        if(cnt=="Yes"):
            continue
        else:
            break

Improved Calculator/Improved Calculator/Improved Calculator.py


T = int(input())
for tc in range(1, T+1):
    s = input().strip()

    dic = {
        'p': 'q',
        'q': 'p',
        'b': 'd',
        'd': 'b',
    }

    result = ''
    for i in range(len(s)-1, -1, -1):
        result += dic[s[i]]

    print('#{} {}'.format(tc, result))


import xlrd
import xlwt
from xlutils.copy import copy
import os

def get_file_and_rewrite(startpath):
    for root, dirs, files in os.walk(startpath):
        for dir in dirs:
            for root, dirs, files in os.walk(startpath + "\\" + dir):
                for file in files:
                    file_loc = root + "\\" + file
                    rb = xlrd.open_workbook(file_loc)
                    wb = copy(rb)
                    read_sheet = rb.sheet_by_index(1)
                    write_sheet = wb.get_sheet(0)
                    execute(read_sheet, write_sheet, file_loc, wb, int(dir))

def check(string):
    try:
        float(string)
        return True
    except ValueError:
        return False

def calTimes(percent, total):
    return round((percent/100)*total)

def execute(read_sheet, write_sheet, filename, wb, total):
    print(total)
    questions = []
    for x in range(read_sheet.ncols):
        answer = []
        for y in range(read_sheet.nrows):
            if check(read_sheet.cell_value(y, x)):
                if y==5 and (sum(answer) + calTimes(read_sheet.cell_value(y, x), total)) < total:
                    answer.append(total - sum(answer))
                else:
                    answer.append(calTimes(read_sheet.cell_value(y, x), total))
        if len(answer)>0:
            questions.append(answer)
    # print(questions)
    for i in range(len(questions)):
        for index, number in enumerate(questions[i]):
            for t in range(int(number)):
                if index==0:
                    row = t + 1
                elif index==1:
                    row = t + questions[i][index-1] + 1
                elif index == 2:
                    row = t + questions[i][index - 1] + questions[i][index - 2] + 1
                elif index == 3:
                    row = t + questions[i][index - 1] + questions[i][index - 2] + questions[i][index-3] + 1
                elif index == 4:
                    row = t + questions[i][index - 1] + questions[i][index - 2] + questions[i][index-3] + questions[i][index-4] + 1

                if int(row) <= total:
                    write_sheet.write(int(row), i, index+1)

    wb.save(filename)

if __name__ == '__main__':
    get_file_and_rewrite('C:\\Users\\Admin\\PycharmProjects\\pythonProject\\venv\\SỐ LIỆU')


class zhang(object):
    pass
s = zhang()
s.name = 'zhang'
print(s.name)

#  可以给实例绑定一个方法 MethodType 这个方法可以给类或者实例绑定一个方法
def set_age(self, age):
    self.age = age

from types import MethodType
s.set_age = MethodType(set_age, s)
s.set_age(24)

# __slots__ 使用方法
# print(dir(zhang)) 查看了一下 里面并没有包括 __slots__ .. 不知道为什么

# 限制实例的属性添加  比方说下面的只允许添加name age 两个属性
class student(object):
    __slots__ = ('name', 'age')   # 用tuple定义允许绑定的属性名称
    pass

s = student()
s.name = "zhang"
s.age = 23
# s.score = 99     # 报错了.

class wang(student):
    pass

g = wang()
g.score = 100
print(g.score)   # 继承以后是允许添加的 ..除非也定义了__slots__

#  @property  (负责把一个方法变成属性调用)

class Person(object):

    @property
    def birth(self):
        return self._birth
    @birth.setter
    def birth(self, value):
        self._birth = value

    @property
    def age(self):
        return 2016 - self._birth

s = Person()
s.birth = 60
print(s.birth)
print(s.age)

#  多重继承

class Money(object):

    def getMethod(self):
        print("you you check not")

class MAndP(Money, Person):
    pass

mp = MAndP()
mp.getMethod()
mp.birth = 1989
print(mp.age)

# 定制类

# __str__ 加上这个就能在调用的时候显示对应的一个文字输出了
# __str__ 这个是用print 打印出来的结果  __repr__ 这个是>>>方式打印的结果显示

class StudentSS(object):

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return 'student object (name: %s)' % self.name

    __repr__ = __str__

print(StudentSS('zhang d'))

# __iter__(迭代对象)  __next__ (下一个返回值)

class Fib(object):
    def __init__(self):
        self.a, self.b = 0, 1

    def __iter__(self):
        return self

    def __next__(self):
        self.a, self.b = self.b, self.a + self.b
        if self.a > 1000:
            raise StopAsyncIteration()
            # pass
        return self.a

# for n in Fib():
#     print(n)

# __getitem__ (表现的想list那样按照下表取出元素) (自定义一个 list dict 等类型时候使用但是没什么事还是少用)
# 与这个对应的还有一个__setitem__() __delitem__() 根据需要的不同进行定制.

class Fibg(object):

    def __getitem__(self, n):

        a, b = 1, 1
        for x in range(n):
            a, b = b, a+b
        return a

fg = Fibg()
print('现在的输出是:', fg[100])

print(list(range(100))[5:10])

# __getattr__  可以用这个来捕获 获取的字符 __getattr__ 这个里面的还可以返回一个函数 调用的时候注意一下 有()

class StudentG(object):

    def __init__(self):
        self.name = 'zhang'

    def __getattr__(self, item):
        print("dont has a attr %s" % item)
        return lambda: 25
        # raise AttributeError('\'Student\' object has no attribute \'%s\'' % item) 可以提示用户并没有这个属性

sg = StudentG()
print(sg.name)
sg.score
print(sg.score())

# 这个__getattr__ 这个可以用来这么去查找和存储一个路径

class Chain(object):
    def __init__(self, path=''):
        self._path = path

    def __getattr__(self, path):
        return Chain('%s/%s' % (self._path, path))

    def __str__(self):
        return self._path

    __repr__ = __str__

print(Chain().status.user.timeline.list)

class ChainL(object):
    def __init__(self, path=''):
        self._path = path

    def currentName(self,user_name):
            return ChainL('%s/users:%s' % (self._path, user_name))

    def __getattr__(self, attr):
        if attr == 'users':            # 这个是查找方法的检测有没有这个方法 然后接受的参数是括号里面的
            return self.currentName    # 以后需要记住这个了 别再犯这个错误了~

        return ChainL("%s/%s" % (self._path, attr))

    def __str__(self):
        return self._path

# print(ChainL().status.users('ZhangSan').timeline.list)
# print(ChainL().status.users.repo)

class MyChainL(object):

    def __init__(self, path=''):
        self._path = path

    def __getattr__(self, attr):
        if attr == 'users':
            return lambda user_name: MyChainL('%susers=%s' % (self._path, user_name))
        return MyChainL("%s/%s" % (self._path, attr))

    def __str__(self):
        return self._path

print(MyChainL().status.users('ZhangSan').timeline.list)

#  上面的还是需要好好理解一下  有一个 递归的概念  有一个 匿名函数的概念  有一个 闭包的概念

# __call__

class StudentC(object):
    def __init__(self, name=''):
        self._name = name

    def __call__(self, *args, **kwargs):
        print('My name is %s' % self._name)

sc = StudentC('zhang')
print(sc())

# 教程上面说 对象和函数 其实本质上的区别就不是那么明显通过__call__ 的方式 直接调用对象相当于调用了__call__函数
# 那么如果要判断 是函数还是对象就需要下面的方式了

print(callable(sc))   # true
print(callable([1, 2, 3])) #False

class AAA(object):
    name = 'AAA'

A = AAA()
print(A.name)

A.name = 'BBB'
print(A.name)

del A.name

print(A.name)


import base64
import os

def img_to_base64(image_bytes):
    """

    :param image_bytes: 文件路径
    :return: base64字符串
    """
    with open(image_bytes, "rb") as fb:
        img = fb.read()
        base64_bytes = base64.b64encode(img)
        img_suffix = os.path.splitext(image_bytes)[1].replace('.', '')
        if img_suffix == 'jpg' or img_suffix == 'JPG' or img_suffix == 'jpeg' or img_suffix == 'JPEG':
            base64_str = 'data:image/jpeg;base64,' + base64_bytes.decode("utf-8")
        elif img_suffix == 'png' or img_suffix == 'PNG':
            base64_str = 'data:image/png;base64,' + base64_bytes.decode("utf-8")
        elif img_suffix == 'gif' or img_suffix == 'GIF':
            base64_str = 'data:image/gif;base64,' + base64_bytes.decode("utf-8")
        elif img_suffix == 'icon' or img_suffix == 'ICON':
            base64_str = 'data:image/x-icon;base64,' + base64_bytes.decode("utf-8")
        else:
            return '不支持此格式文件'
    return base64_str


from pathlib import Path
import re
from collections import deque
data_folder = Path(".").resolve()
reg = re.compile(r"(\d+) <-> (.+)")

def find_groups(comms):
    groups = []
    for i in range(len(comms)):
        in_groups = []
        for j in range(len(groups)):
            for k in range(len(comms[i])):
                if comms[i][k] in groups[j]:
                    in_groups.append(j)
                    break
        if not in_groups:
            groups.append(comms[i])
        else:
            for red_group in in_groups[-1:0:-1]:
                groups[in_groups[0]] += groups[red_group]
                del groups[red_group]
            groups[in_groups[0]] += comms[i]
            groups[in_groups[0]] = list(set(groups[in_groups[0]]))
    return groups

def main():
    data = data_folder.joinpath("input.txt").read_text()
    direct_comm = []
    for line in data.split("\n"):
        m = reg.match(line)
        l = [int(m.group(1))]
        l += l+[int(d) for d in m.group(2).split(", ")]
        l = list(set(l))
        direct_comm.append(l)

    print("Part 1")
    groups = find_groups(direct_comm)
    prog_id = 0
    len_prog_id_group = None
    for group in groups:
        if prog_id in group:
            len_prog_id_group = len(group)
            break

    print(f"There are {len_prog_id_group} programs in the group that contains program ID {prog_id}")
    print()
    print("Part 2")
    print(f"There are {len(groups)} program groups in total")

if __name__ == "__main__":
    main()


from collections import defaultdict,deque
def convert(n,k):
    ret = ''
    while n:
        ret += str(n%k)
        n//=k
    return ret[::-1]

def isPrime(num):
    if num==2 or num==3: return True
    if num%2 == 0 or num%3==0 or num<2 : return False
    for i in range(3, int(num**.5)+1, 2):
        if num%i == 0: return False
    return True

def solution(n, k):
    ans = 0
    st = convert(n,k).split('0')
    for s in st:
        if s == "": continue
        if isPrime(int(s)): ans += 1

    return ans

x_k진수에서소수개수구하기.py


'''
Created on 25 abr. 2020

@author: jesus.fernandez
'''
import urllib3
import pandas as pd

class WorldCovidDataCrawler(object):
    '''
    classdocs
    '''
    #Declaración de constantes
    URL_BASE = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/'
    URL_CONFIRMED = URL_BASE+ 'time_series_covid19_confirmed_global.csv'
    URL_DEATH = URL_BASE+'time_series_covid19_deaths_global.csv'
    URL_RECOVERED = URL_BASE+'time_series_covid19_recovered_global.csv'

    data_recuperados = None
    data_fallecidos = None
    data_confirmados = None

    def __init__(self):
        '''
        Constructor
        '''

    def extract(self):
        #Se obtienen los datos de recuperados, fallecidos y confirmados
        self.data_recuperados = pd.read_csv(WorldCovidDataCrawler.URL_RECOVERED)
        self.data_fallecidos = pd.read_csv(WorldCovidDataCrawler.URL_DEATH)
        self.data_confirmados = pd.read_csv(WorldCovidDataCrawler.URL_CONFIRMED)

    def transform(self):
        self.data_recuperados['region'] = self.data_recuperados['Province/State'].fillna('')+self.data_recuperados['Country/Region'].fillna('')
        self.data_recuperados = self.data_recuperados.melt(id_vars=['Province/State', 'Country/Region', 'Lat', 'Long','region'],
                                            var_name="fecha",
                                            value_name="numero")
        self.data_recuperados['fecha'] = self.data_recuperados['fecha'].astype('datetime64[ns]')

        self.data_fallecidos['region'] = self.data_fallecidos['Province/State'].fillna('')+self.data_fallecidos['Country/Region'].fillna('')
        self.data_fallecidos = self.data_fallecidos.melt(id_vars=['Province/State', 'Country/Region', 'Lat', 'Long','region'],
                                            var_name="fecha",
                                            value_name="numero")
        self.data_fallecidos['fecha'] = self.data_fallecidos['fecha'].astype('datetime64[ns]')

        self.data_confirmados['region'] = self.data_confirmados['Province/State'].fillna('')+self.data_confirmados['Country/Region'].fillna('')
        self.data_confirmados = self.data_confirmados.melt(id_vars=['Province/State', 'Country/Region', 'Lat', 'Long','region'],
                                            var_name="fecha",
                                            value_name="numero")
        self.data_confirmados['fecha'] = self.data_confirmados['fecha'].astype('datetime64[ns]')

    def load(self):
        from common.SQLUtil import SQLUtil

        SQLUtil.pandasToTable(self.data_recuperados,'covid_19_recuperados')
        SQLUtil.pandasToTable(self.data_fallecidos,'covid_19_fallecidos')
        SQLUtil.pandasToTable(self.data_confirmados,'covid_19_confirmados')

    def exec(self):
        self.extract()
        self.transform()
        self.load()


def min_num():
    lis=[2,3,4,5]
    min = lis[0]
    for x in lis:
        if x < min:
            min = x
    print("minimum number in list",min)

if __name__  == '__main__':
    min_num()


import pymem
import pymem.process
import psutil
import os
import time
class PymongUS:
    def __init__(self):
        self.dwspeed = 0x00DA3C30
        self.imposter =0x00DA5A84
        self.pm = pymem.Pymem('Among Us.exe')

    def makemeImposter(self):
        try:
            client = pymem.process.module_from_name(self.pm.process_handle,"GameAssembly.dll").lpBaseOfDll
            imp = self.pm.read_int(client+self.imposter)
            #print(hex(imp))
            off1 = self.pm.read_int(imp+0x5C)
            off2 = self.pm.read_int(off1+0x0)
            off3 = self.pm.read_int(off2+0x034)
            off4 = hex(off3+0x28)
            intoff4 = int(off4,16)
            self.pm.write_int(intoff4,1)
            return {"error":False,"Imposter":True}
        except Exception:
            return {"error":True,"Imposter":False}

    def changeSpeed(self,aspeed):
        try:
            client = pymem.process.module_from_name(self.pm.process_handle,"GameAssembly.dll").lpBaseOfDll
            speed = self.pm.read_int(client+self.dwspeed)
            off1 = self.pm.read_int(speed+0x5C)
            off2 = self.pm.read_int(off1+0x14)
            off3 = hex(off2+0x14)
            intva = int(off3, 16)
            self.pm.write_float(intva,float(aspeed))
            return {"error":False,"Speed":True}
        except Exception:
            return {"error":True,"Speed":False}

def checkIfProcessRunning(processName):
    '''
    Check if there is any running process that contains the given name processName.
    I was so bored i stole this function from :
    https://gist.github.com/Sanix-Darker/8cbed2ff6f8eb108ce2c8c51acd2aa5a
    '''
    #Iterate over the all the running process
    for proc in psutil.process_iter():
        try:
            # Check if process name contains the given name string.
            if processName.lower() in proc.name().lower():
                return True
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass
    return False

while True:
    if checkIfProcessRunning('Among Us'):
        print('[+] Among US Is Running\n[+]Initializing Pymong Please Wait...........')
        time.sleep(10)
        os.system('cls' if os.name == 'nt' else 'clear')
        break
    else:
        os.system('cls' if os.name == 'nt' else 'clear')
        print('[+] Waiting For Among US')

def intro():
    print("""
        ______                                  _   _ _____
    | ___ \                                | | | /  ___|
    | |_/ /   _ _ __ ___   ___  _ __   __ _| | | \ `--.
    |  __/ | | | '_ ` _ \ / _ \| '_ \ / _` | | | |`--. \
    | |  | |_| | | | | | | (_) | | | | (_| | |_| /\__/ /
    \_|   \__, |_| |_| |_|\___/|_| |_|\__, |\___/\____/
        __/ |                       __/ |
        |___/                       |___/

            Python Sample Code For Memory Reading/Writing\n\n""")

if __name__ == "__main__":
    intro()
    app = PymongUS()
    print("[1] Change Speed \n[2] Access Imposter Feature : ")
    cheatOptions = int(input("Select your option either 1 or two : "))
    if cheatOptions == 1:
        speedtoset = float(input("        [+]Select Your Desired Speed : "))
        app.changeSpeed(speedtoset)
        print(f'        [+] Speed Changed!! Current Speed IS : {speedtoset}')
    elif cheatOptions == 2:
        app.makemeImposter()
        print('        [+] You Now Have Imposter Access')


# -*- coding: utf-8 -*-
# __author__ = 'XingHuan'
# 4/15/2018

import logging
import inspect
import functools
import time

log_level = logging.DEBUG
log_formats = [
    '%(levelname)s:%(name)s: %(asctime)s %(pathname)s[line:%(lineno)d] %(funcName)s %(message)s',
    '%(levelname)s:%(name)s: %(asctime)s %(message)s',
]
logger_list = {}

# logging.basicConfig(level=log_level)

def get_logger(name, format=0):
    global logger_list

    if name not in logger_list:
        logger = logging.getLogger(name)
        logger.setLevel(log_level)
        date_fmt = '%d %b %Y %H:%M:%S'
        str_fmt = log_formats[format]
        formatter = logging.Formatter(str_fmt, date_fmt)
        ch_handler = logging.StreamHandler()
        ch_handler.setFormatter(formatter)
        logger.addHandler(ch_handler)
        logger_list[name] = logger
    else:
        logger = logger_list[name]
    return logger

time_logger = get_logger('log_cost_time', 1)

def log_cost_time(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        # logger = get_logger('log_cost_time', 1)
        time_logger.debug('Enter:%s %s' % (inspect.getmodule(func).__name__, func.__name__))
        start = time.time()
        u = func(*args, **kwargs)
        time_logger.debug('Leave:%s cost time: %s' % (func.__name__, time.time() - start))
        return u
    return wrapper

def get_current_function_name():
    return inspect.stack()[1][3]


import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense

X = np.array([[185.32,  12.69],
    [259.92,  11.87],
    [231.01,  14.41],
    [175.37,  11.72],
    [187.12,  14.13]])

Y = np.array([[1.],
 [0.],
 [0.],
 [0.],
 [1.]])

print(f"Temperature Max, Min pre normalization: {np.max(X[:,0]):0.2f}, {np.min(X[:,0]):0.2f}")
print(f"Duration    Max, Min pre normalization: {np.max(X[:,1]):0.2f}, {np.min(X[:,1]):0.2f}")
norm_l = tf.keras.layers.Normalization(axis=-1)
norm_l.adapt(X)  # learns mean, variance
Xn = norm_l(X)
print(f"Temperature Max, Min post normalization: {np.max(Xn[:,0]):0.2f}, {np.min(Xn[:,0]):0.2f}")
print(f"Duration    Max, Min post normalization: {np.max(Xn[:,1]):0.2f}, {np.min(Xn[:,1]):0.2f}")

Xt = np.tile(Xn,(1000,1))
Yt= np.tile(Y,(1000,1))
print(Xt.shape, Yt.shape)

tf.random.set_seed(1234)  # applied to achieve consistent results
model = Sequential(
    [
        tf.keras.Input(shape=(2,)),
        Dense(3, activation='sigmoid', name = 'layer1'),
        Dense(1, activation='sigmoid', name = 'layer2')
     ]
)

model.summary()

L1_num_params = 2 * 3 + 3   # W1 parameters  + b1 parameters
L2_num_params = 3 * 1 + 1   # W2 parameters  + b2 parameters
print("L1 params = ", L1_num_params, ", L2 params = ", L2_num_params  )

W1, b1 = model.get_layer("layer1").get_weights()
W2, b2 = model.get_layer("layer2").get_weights()
print(f"W1{W1.shape}:\n", W1, f"\nb1{b1.shape}:", b1)
print(f"W2{W2.shape}:\n", W2, f"\nb2{b2.shape}:", b2)

model.compile(
    loss = tf.keras.losses.BinaryCrossentropy(),
    optimizer = tf.keras.optimizers.Adam(learning_rate=0.01),
)

model.fit(
    Xt,Yt,
    epochs=10,
)

W1, b1 = model.get_layer("layer1").get_weights()
W2, b2 = model.get_layer("layer2").get_weights()
print("W1:\n", W1, "\nb1:", b1)
print("W2:\n", W2, "\nb2:", b2)

W1 = np.array([
    [-8.94,  0.29, 12.89],
    [-0.17, -7.34, 10.79]] )
b1 = np.array([-9.87, -9.28,  1.01])
W2 = np.array([
    [-31.38],
    [-27.86],
    [-32.79]])
b2 = np.array([15.54])
model.get_layer("layer1").set_weights([W1,b1])
model.get_layer("layer2").set_weights([W2,b2])

X_test = np.array([
    [200,13.9],  # postive example
    [200,17]])   # negative example
X_testn = norm_l(X_test)
predictions = model.predict(X_testn)
print("predictions = \n", predictions)


# https://leetcode.com/problems/plus-one/
# 66. Plus One
# You are given a large integer represented as an integer array digits,
# where each digits[i] is the ith digit of the integer.
# The digits are ordered from most significant to least significant in left-to-right order.
# The large integer does not contain any leading 0's.
# Increment the large integer by one and return the resulting array of digits.

# Example 1:
# Input: digits = [1,2,3]
# Output: [1,2,4]
# Explanation: The array represents the integer 123.
# Incrementing by one gives 123 + 1 = 124.
# Thus, the result should be [1,2,4].

# Example 2:
# Input: digits = [4,3,2,1]
# Output: [4,3,2,2]
# Explanation: The array represents the integer 4321.
# Incrementing by one gives 4321 + 1 = 4322.
# Thus, the result should be [4,3,2,2]

# Example 3:
# Input: digits = [9]
# Output: [1,0]
# Explanation: The array represents the integer 9.
# Incrementing by one gives 9 + 1 = 10.
# Thus, the result should be [1,0].

# Constraints:
# 1 <= digits.length <= 100
# 0 <= digits[i] <= 9
# digits does not contain any leading 0's.

from ast import List
from typing import List

class PlusOne:
    def plusOne(self, digits: List[int]) -> List[int]:
        lenDigi = len(digits)
        if digits[lenDigi - 1] < 9:
            digits[lenDigi - 1] += 1
            return digits
        i = lenDigi - 1
        while digits[i] + 1 == 10 and i >= 0:
            digits[i] = 0
            i -= 1
        if i >= 0:
            digits[i] += 1
            return digits
        digits.insert(0, 1)
        return digits

# Testing
instance = PlusOne()
digits = [9, 9, 9]

print("digits before adding 1: ")
print(digits)
print("digits after adding 1: ")
print(instance.plusOne(digits))


import sys
sys.stdin = open("B3_input.txt")

# def process_solution(a, k, sum):
#     global nmin
#     if sum > nmin : return
#
# def make_candidates(a, k, input, c):
#     in_perm = [0] * NMAX
#
#     for i in range(1, k):
#         in_perm[a[i]] = 1
#
#     ncands = 0
#     for i in range(1, input+1):
#         if in_perm[i] == 0:
#             c[ncands] = i
#             ncands += 1
#
#     return ncands
#
# def backtrack(a, k, input, sum):
#     global nmin
#
#     if sum > nmin: return
#
#     global MAXCANDIDATES
#     c = [0] * MAXCANDIDATES
#
#     if k == input:
#         process_solution(a, k, sum)
#         if sum < nmin:
#             nmin = sum
#             sum=0
#     else:
#         k += 1
#         ncands = make_candidates(a, k, input, c)
#         for i in range(ncands):
#             a[k] = c[i]
#             backtrack(a, k, input, sum + arr[k-1][a[k]-1])
#
# n = int(input())
# arr = [[0 for _ in range(n)] for _ in range(n)]
# for i in range(n):
#     arr[i] = list(map(int, input().split()))
#
# MAXCANDIDATES = 100
# NMAX = 100
# data = [0]
# nmin = 99999
# for i in range(n):
#     data.append(i)
# a = [0] * NMAX
# backtrack(a, 0, n, 0)
# # minres = []
# # for i in range(n):
# #     minres.append(min(arr[i]))
# # print(sum(minres))
# print(nmin)

def dfs1(no, nsum):
    global sol
    if nsum > sol : return # 가지치기
    if no >= n:
        # for i in range(n): print(rec[i], end=" ")
        # print(nsum)
        if nsum<sol:
            sol = nsum
        return
    # 현재 행에 모든 열의 수를 고르는 경우의 수 시도
    for i in range(n):
        rec[no] = arr[no][i]
        dfs1(no+1, nsum + arr[no][i])
    # print(sol)

def dfs2(no, nsum):
    global sol
    if nsum > sol : return # 가지치기
    if no >= n:
        # for i in range(n): print(rec[i], end=" ")
        # print(nsum)
        if nsum < sol:
            sol = nsum
        return
    # 현재 행에 모든 열의 수를 고르는 경우의 수 시도
    for i in range(n):
        if chk[i] == 1:continue
        chk[i] = 1
        rec[no] = arr[no][i]
        dfs2(no+1, nsum + arr[no][i])
        chk[i] = 0

n = int(input())
arr = [[0 for _ in range(n)] for _ in range(n)]
for i in range(n):
    arr[i] = list(map(int, input().split()))

sol = 100000
chk = [0] * 11
rec = [0] * 11

dfs1(0,0)
print(sol)
sol = 100000
dfs2(0,0)
print(sol)


import function_rat_num as fr
import sys

def x():
    global first_num
    first_num = float(input('Введите первое число: ').replace(',', '.'))
    return first_num

def y():
    global second_num
    second_num = float(input('Введите второе число: ').replace(',', '.'))
    return second_num

def select_operation():
    global operation
    operation = (input(f'Выбирите действие: +, -, *, /: '))
    if operation == '+' or '-' or '/' or '*':
        return operation
    else:
        print('Неправильный ввод')

def res(first_num, second_num):
        if  operation == '+':
            res = first_num + second_num
            result = round(res, 3)
            return result
        elif operation == '-':
            res = first_num - second_num
            result = round(res, 3)
            return result
        elif operation == '*':
            res = first_num * second_num
            result = round(res, 3)
            return result
        elif operation == '/':
            res = first_num / second_num
            result = round(res, 3)
            return result
        else:
            print('Неправильный ввод')

def main_terminal():
    global file
    x = fr.x()
    while True:
        y = fr.y()
        oper = fr.select_operation()
        res = fr.res(x, y)
        file = 'results.txt'
        with open('results.txt', 'a') as data:
            data.write(f'Результат {x} {oper} {y} = {res}\n')
        print(f'Результат действия {x} {oper} {y} = {res}\n(Сделаем запись в фай с результатами)' )
        again = input('Использовать калькулятор еще раз? Yes/No: ').lower()
        if again == 'yes':
            useresult = input('ВЫ хотите использовать последнее дейстиве? (Yes/No): ').lower()
            if useresult == 'yes':
                x = res
                continue
            elif useresult == 'no':
                break
            else:
                sys.exit()
        else:
                sys.exit()


import pandas as pd
import numpy as np
import os
import random
import pywt
import pickle
from tqdm import tqdm
import torch
from random import sample
from dataloaders.utils import Normalizer,components_selection_one_signal,mag_3_signals,PrepareWavelets,FiltersExtention
from sklearn.utils import class_weight
from skimage.transform import resize
freq1 = 0.3
freq2 = 20

# ========================================       Data loader Base class               =============================
class BASE_DATA():

    def __init__(self, args):
        """
        root_path                      : Root directory of the data set
        freq_save_path                 : The path to save genarated Spectrogram. If the file has already been generated, Load it directly
        window_save_path               : The path to save genarated Window index. If the file has already been generated, Load it directly
                                         This could save time by avoiding generate them again.

        data_name (Str)                : the name of data set
                                       --->[TODO]

        freq (int)                     :Sampling Frequency of the correponding dataset
        representation_type (Str)      :  What kind of representation should be load
                                       --->[time, freq, time_freq]

        difference  (Bool)             : Whether to calculate the first order derivative of the original data
        datanorm_type (Str)            : How to normalize the data
                                       --->[standardization, minmax, per_sample_std, per_sample_minmax]

        load_all (Bool)                : This is for Freq representation data. Whether load all files in one time. this could save time by training, but it needs a lot RAM
        train_vali_quote (float)       : train vali split quote , default as 0.8

        windowsize                     :  the size of Sliding Window
        -------------------------------------------------------
		if training mode, The sliding step is 50% of the windowsize
        if test mode, The step is 10% of the windowsize. (It should be as one, But it results in to many window samples, it is difficult to generate the spectrogram)
        -------------------------------------------------------
        drop_transition  (Bool)        : Whether to drop the transition parts between different activities
        wavelet_function (Str)         : Method to generate Spectrogram
                                       ---> []

        """
        self.root_path              = args.root_path
        self.freq_save_path         = args.freq_save_path
        self.window_save_path       = args.window_save_path
        self.data_name              = args.data_name

        window_save_path = os.path.join(self.window_save_path,self.data_name)
        if not os.path.exists(window_save_path):
            os.mkdir(window_save_path)
        self.window_save_path       = window_save_path
        self.representation_type    = args.representation_type
        #assert self.data_name in []
        self.freq                   = args.sampling_freq

        self.difference             = args.difference
        self.filtering              = args.filtering
        self.magnitude              = args.magnitude
        self.datanorm_type          = args.datanorm_type
        self.load_all               = args.load_all
        self.train_vali_quote       = args.train_vali_quote
        self.windowsize             = args.windowsize
        self.drop_transition        = args.drop_transition
        self.wavelet_function       = args.wavelet_function
        #self.wavelet_filtering      = args.wavelet_filtering
        #self.number_wavelet_filtering = args.number_wavelet_filtering

        # ======================= Load the Data =================================
        self.data_x, self.data_y = self.load_all_the_data(self.root_path)
        # data_x : sub_id, sensor_1, sensor_2,..., sensor_n , sub
        # data_y : activity_id   index:sub_id
        # update col_names
        self.col_names = list(self.data_x.columns)[1:-1]

        # ======================= Differencing the Data =================================
        if self.difference:
            print("Channel Augmentation : Differencing")
            self.data_x = self.differencing(self.data_x.set_index('sub_id').copy())

        # data_x : sub_id, sensor_1, sensor_2,..., sensor_n , sub

        # ======================== Filtering the Data =========================================
        if self.filtering:
            print("Channel Augmentation : Acc Gyro Filtering")
            self.data_x = self.Sensor_data_noise_grav_filtering(self.data_x.set_index('sub_id').copy())

        # ======================== Mag the Data =====================================

        if self.magnitude:
            print("Channel Augmentation : Magnitute Calculating for acc and Gyro")
            self.data_x.columns , columns_groups = self.regroup_and_reindex_all_cols(self.data_x.set_index('sub_id').copy())

            temp_columns = list(self.data_x.columns[1:-1])
            for cols in columns_groups:
                if len(cols)== 3:
                    col1, col2, col3 = cols
                    col = "_".join(col1.split("_")[:-1])+"_mag"
                    temp_columns.append(col)
                    self.data_x[col] = mag_3_signals(np.array(self.data_x[col1]),
                                                     np.array(self.data_x[col2]),
                                                     np.array(self.data_x[col3]))
            self.data_x = self.data_x[["sub_id"]+temp_columns+["sub"] ]

        # ======================= Generate the Sliding window for Train/Test =================================
        # train/test is different by the sliding step.
        self.train_slidingwindows = self.get_the_sliding_index(self.data_x.copy(), self.data_y.copy(), "train")
        if self.exp_mode not in ["SOCV","FOCV"]:
            self.test_slidingwindows  = self.get_the_sliding_index(self.data_x.copy(), self.data_y.copy(), "test")

        # ----------------------- TODO ----------------------------------------
        if self.representation_type in ["freq", "time_freq"]:
            print("------------freq representation is needed -----------------")
            assert self.freq_save_path is not None
            # train/test is different by the sliding step.
            self.genarate_spectrogram(flag = "train")
            self.genarate_spectrogram(flag = "test")

            if self.load_all:
                print("-------load all freq DATA --------")
                self.data_freq = {}
                for file in tqdm(self.train_freq_file_name):
                    with open(os.path.join(self.train_freq_path,"{}.pickle".format(file)), 'rb') as handle:
                        sample_x = pickle.load(handle)
                    self.data_freq[file] = sample_x

        # ----------------------- TODO ----------------------------------------
        if self.exp_mode in ["SOCV","FOCV"]:
            self.num_of_cv = 5
            self.index_of_cv = 0
            self.step = int(len(self.train_slidingwindows)/self.num_of_cv)
            self.window_index_list = list(np.arange(len(self.train_slidingwindows)))
            random.shuffle(self.window_index_list)
            if self.datanorm_type is not None:
                self.normalized_data_x = self.normalization(self.data_x.copy())
            else:
                self.normalized_data_x = self.data_x.copy()

        elif self.exp_mode == "LOCV":
            self.num_of_cv = len(self.LOCV_keys)
            self.index_of_cv = 0

        else:
            self.num_of_cv = 1

    def update_train_val_test_keys(self):
        """
        It should be called at the begin of each iteration
        it will update:
        1. train_window_index
        2. vali_window_index
        3. test_window_index
        it will also:
        normalize the data , because each iteration uses different training data
        calculate the weights of each class
        """
        if self.exp_mode in ["Given", "LOCV"]:
            if self.exp_mode == "LOCV":
                print("Leave one Out Experiment : The {} Part as the test".format(self.index_of_cv+1))

                self.test_keys =  self.LOCV_keys[self.index_of_cv]
                self.train_keys = [key for key in self.all_keys if key not in self.test_keys]
                # update the index_of_cv for the next iteration
                self.index_of_cv = self.index_of_cv + 1

            # Normalization the data
            if self.datanorm_type is not None:
                train_vali_x = pd.DataFrame()
                for sub in self.train_keys:
                    temp = self.data_x[self.data_x[self.split_tag]==sub]
                    train_vali_x = pd.concat([train_vali_x,temp])

                test_x = pd.DataFrame()
                for sub in self.test_keys:
                    temp = self.data_x[self.data_x[self.split_tag]==sub]
                    test_x = pd.concat([test_x,temp])

                train_vali_x, test_x = self.normalization(train_vali_x, test_x)

                self.normalized_data_x = pd.concat([train_vali_x,test_x])
                self.normalized_data_x.sort_index(inplace=True)
            else:
                self.normalized_data_x = self.data_x.copy()

            # 根据test的keys  筛选出 window的第一个element有哪些
            all_test_keys = []
            if self.split_tag == "sub":
                for sub in self.test_keys:
                    all_test_keys.extend(self.sub_ids_of_each_sub[sub])
            else:
                all_test_keys = self.test_keys.copy()

            # -----------------test_window_index---------------------
            test_file_name = os.path.join(self.window_save_path,
                                          "{}_droptrans_{}_windowsize_{}_{}_test_ID_{}.pickle".format(self.data_name,
                                                                                self.drop_transition,
                                                                                self.exp_mode,
                                                                                self.windowsize,
                                                                                self.index_of_cv-1))
            if os.path.exists(test_file_name):
                with open(test_file_name, 'rb') as handle:
                    self.test_window_index = pickle.load(handle)
            else:
                self.test_window_index = []
                for index, window in enumerate(self.test_slidingwindows):
                    sub_id = window[0]
                    if sub_id in all_test_keys:
                        self.test_window_index.append(index)
                with open(test_file_name, 'wb') as handle:
                    pickle.dump(self.test_window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

            # -----------------train_vali_window_index---------------------

            train_file_name = os.path.join(self.window_save_path,
                                           "{}_droptrans_{}_windowsize_{}_{}_train_ID_{}.pickle".format(self.data_name,
                                                                                self.drop_transition,
                                                                                self.exp_mode,
                                                                                self.windowsize,
                                                                                self.index_of_cv-1))
            if os.path.exists(train_file_name):
                with open(train_file_name, 'rb') as handle:
                    train_vali_window_index = pickle.load(handle)
            else:
                train_vali_window_index = []
                for index, window in enumerate(self.train_slidingwindows):
                    sub_id = window[0]
                    if sub_id not in all_test_keys:
                        train_vali_window_index.append(index)
                with open(train_file_name, 'wb') as handle:
                    pickle.dump(train_vali_window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

            random.shuffle(train_vali_window_index)
            self.train_window_index = train_vali_window_index[:int(self.train_vali_quote*len(train_vali_window_index))]
            self.vali_window_index = train_vali_window_index[int(self.train_vali_quote*len(train_vali_window_index)):]

        elif self.exp_mode in ["SOCV","FOCV"]:
            print("Overlapping random Experiment : The {} Part as the test".format(self.index_of_cv+1))
            start = self.index_of_cv * self.step
            if self.index_of_cv < self.num_of_cv-1:
                end = (self.index_of_cv+1) * self.step
            else:
                end = len(self.train_slidingwindows)

            train_vali_index = self.window_index_list[0:start] + self.window_index_list[end:len(self.window_index_list)]
            self.test_window_index = self.window_index_list[start:end]
            # copy shuffle
            self.train_window_index = train_vali_index[:int(self.train_vali_quote*len(train_vali_index))]
            self.vali_window_index = train_vali_index[int(self.train_vali_quote*len(train_vali_index)):]

            self.index_of_cv = self.index_of_cv + 1

        else:
            raise NotImplementedError

        self.act_weights = self.update_classes_weight()

    def update_classes_weight(self):
        class_transform = {x: i for i, x in enumerate(self.no_drop_activites)}

        y_of_all_windows  = []
        # get all labels of all windows
        for index in self.train_window_index:

            start_index = self.train_slidingwindows[index][1]
            end_index = self.train_slidingwindows[index][2]

            y_of_all_windows.append(class_transform[self.data_y.iloc[start_index:end_index].mode().loc[0]])
        y_of_all_windows = np.array(y_of_all_windows)

        target_count = np.array([np.sum(y_of_all_windows == label) for label in set(y_of_all_windows)])
        weight_target = 1.0 / target_count
        weight_samples = np.array([weight_target[t] for t in y_of_all_windows])
        weight_samples = torch.from_numpy(weight_samples)
        weight_samples = weight_samples.double()

        print("[-] Target sampling weights: ", weight_target)

        return weight_samples

    def load_all_the_data(self, root_path):
        raise NotImplementedError

    def Sensor_data_noise_grav_filtering(self, df):
        """
        df : sensor_1, sensor_2, sub
           index=sub_id
        """
        all_columns = list(df.columns)[:-1]
        #rest_columns = list(set(all_columns) - set(self.col_names))

        filtered_data = []
        for sub_id in df.index.unique():
            temp = df.loc[sub_id,all_columns]
            filtered_temp = pd.DataFrame()

            for col in temp.columns:
                t_signal=np.array(temp[col]) # copie the signal values in 1D numpy array

                if 'acc' in col and "diff" not in col:
                    # the 2nd output DC_component is the gravity_acc
                    # The 3rd one is the body_component which in this case the body_acc
                    grav_acc,body_acc=components_selection_one_signal(t_signal,freq1,freq2,self.freq) # apply components selection

                    filtered_temp[col]=body_acc
                    filtered_temp['grav'+col]= grav_acc

                elif 'gyro' in col and "diff" not in col:

                    # The 3rd output of components_selection is the body_component which in this case the body_gyro component
                    _,        body_gyro=components_selection_one_signal(t_signal,freq1,freq2,self.freq)  # apply components selection
                    filtered_temp[col]=body_gyro # t_body_acc storing with the appropriate axis selected
                else:

                    filtered_temp[col] = t_signal

            #filtered_temp = filtered_temp[sorted(list(filtered_temp.columns))]
            filtered_temp.index = temp.index
            filtered_data.append(filtered_temp)

        filtered_data = pd.concat(filtered_data)
        #filtered_data = pd.concat([df.loc[:,rest_columns], filtered_data], axis=1)
        #filtered_data = filtered_data[sorted(list(filtered_data.columns))]
        filtered_data = pd.concat([filtered_data, df.iloc[:,-1]], axis=1)

        return filtered_data.reset_index()

    def differencing(self, df):
        # columns = [, "acc_x"..."acc_y", "sub"], index is sub_id
        # define the name for differenced columns
        all_columns = list(df.columns)[:-1]
        rest_columns = list(set(all_columns) - set(self.col_names))

        columns = ["diff_"+i for i in self.col_names]

        # The original data has been divided into segments by sub_id: a segment belongs to a same user
        # There is no continuity between different segments, so diffrecne is only done within each segment

        # ALL data Diff
        diff_data = []
        for id in df.index.unique():
            diff_data.append(df.loc[id,self.col_names].diff())
        diff_data = pd.concat(diff_data)
        diff_data.columns = columns
        diff_data.fillna(method ="backfill",inplace=True)

        data = pd.concat([df.iloc[:,:-1],  diff_data], axis=1)
        #data = data[sorted(list(data.columns))]
        data = pd.concat([data, df.iloc[:,-1]], axis=1)

        return data.reset_index()

    def normalization(self, train_vali, test=None):
        train_vali_sensors = train_vali.iloc[:,1:-1]
        self.normalizer = Normalizer(self.datanorm_type)
        self.normalizer.fit(train_vali_sensors)
        train_vali_sensors = self.normalizer.normalize(train_vali_sensors)
        train_vali_sensors = pd.concat([train_vali.iloc[:,0],train_vali_sensors,train_vali.iloc[:,-1]], axis=1)
        if test is None:
            return train_vali_sensors
        else:
            test_sensors  = test.iloc[:,1:-1]
            test_sensors  = self.normalizer.normalize(test_sensors)
            test_sensors  =  pd.concat([test.iloc[:,0],test_sensors,test.iloc[:,-1]], axis=1)
            return train_vali_sensors, test_sensors

    def get_the_sliding_index(self, data_x, data_y , flag = "train"):
        """
        Because of the large amount of data, it is not necessary to store all the contents of the slidingwindow,
        but only to access the index of the slidingwindow
        Each window consists of three parts: sub_ID , start_index , end_index
        The sub_ID ist used for train test split, if the subject train test split is applied
        """
        if os.path.exists(os.path.join(self.window_save_path,
                                       "{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name,
                                                                                flag,
                                                                                self.drop_transition,
                                                                                self.windowsize))):
            print("-----------------------Sliding file are generated -----------------------")
            with open(os.path.join(self.window_save_path,
                                   "{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name,
                                                                                flag,
                                                                                self.drop_transition,
                                                                                self.windowsize)), 'rb') as handle:
                window_index = pickle.load(handle)
        else:
            print("----------------------- Get the Sliding Window -------------------")

            data_y = data_y.reset_index()
            data_x["activity_id"] = data_y["activity_id"]

            if self.drop_transition:
                data_x['act_block'] = ((data_x['activity_id'].shift(1) != data_x['activity_id']) | (data_x['sub_id'].shift(1) != data_x['sub_id'])).astype(int).cumsum()
            else:
                data_x['act_block'] = (data_x['sub_id'].shift(1) != data_x['sub_id']).astype(int).cumsum()

            freq         = self.freq
            windowsize   = self.windowsize

            if flag == "train":
                displacement = int(0.5 * self.windowsize)
                #drop_for_augmentation = int(0.2 * self.windowsize)
            elif flag == "test":
                displacement = int(0.1 * self.windowsize)
                #drop_for_augmentation = 1

            window_index = []
            for index in data_x.act_block.unique():

                temp_df = data_x[data_x["act_block"]==index]
                assert len(temp_df["sub_id"].unique()) == 1
                sub_id = temp_df["sub_id"].unique()[0]
                start = temp_df.index[0]# + drop_for_augmentation
                end   = start+windowsize

                while end <= temp_df.index[-1]+1:# + drop_for_augmentation :

                    if temp_df.loc[start:end-1,"activity_id"].mode().loc[0] not in self.drop_activities:
                        window_index.append([sub_id, start, end])

                    start = start + displacement
                    end   = start + windowsize

            with open(os.path.join(self.window_save_path,"{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name, flag, self.drop_transition,windowsize)), 'wb') as handle:
                pickle.dump(window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

        return window_index

    def genarate_spectrogram(self, flag="train"):
        save_path = os.path.join(self.freq_save_path,self.data_name)
        if not os.path.exists(save_path):
            os.mkdir(save_path)

        if flag == "train":
            displacement = int(0.5 * self.windowsize)
            slidingwindows = self.train_slidingwindows
            self.train_freq_path = os.path.join(save_path,"diff_{}_window_{}_step_{}_drop_trans_{}".format(self.difference, self.windowsize,displacement, self.drop_transition))
            freq_path = self.train_freq_path
        elif flag == "test":
            displacement = int(0.1 * self.windowsize)
            slidingwindows = self.test_slidingwindows
            self.test_freq_path = os.path.join(save_path,"diff_{}_window_{}_step_{}_drop_trans_{}".format(self.difference, self.windowsize, displacement, self.drop_transition))
            freq_path = self.test_freq_path

        if os.path.exists(freq_path):
            print("----------------------- file are generated -----------------------")
            if flag == "train":
                with open(os.path.join(freq_path,"freq_file_name.pickle"), 'rb') as handle:
                    self.train_freq_file_name = pickle.load(handle)
            else:
                with open(os.path.join(freq_path,"freq_file_name.pickle"), 'rb') as handle:
                    self.test_freq_file_name = pickle.load(handle)

        else:
            print("----------------------- spetrogram generating for {} -----------------------".format(flag))
            os.mkdir(freq_path)

            scales1 = np.arange(1, self.freq + 1)

            totalscal = self.freq
            fc = pywt.central_frequency(self.wavename)#计算小波函数的中心频率
            cparam = 2 * fc * totalscal  #常数c
            scales2 = cparam/np.arange(totalscal,0,-1) #为使转换后的频率序列是一等差序列，尺度序列必须取为这一形式（也即小波尺度）

            if self.windowsize >= 60 and "synthetic" not in self.data_name:
                l_scale = 2
            else:
                l_scale = 1

            if self.freq >=40 and "synthetic" not in self.data_name:
                f_scale = 2
            else:
                f_scale = 1

            if f_scale>1 or l_scale>1:
                resize_flag = True
            else:
                resize_flag = False

            freq_file_name = []

            temp_data = self.normalization(self.data_x.copy())
            for window in slidingwindows:
                sub_id = window[0]
                start_index = window[1]
                end_index = window[2]

                name = "{}_{}_{}".format(sub_id,start_index,end_index)
                freq_file_name.append(name)

                sample_x = temp_data.iloc[start_index:end_index,1:-1].values
                scalogram = []

                for j in range(sample_x.shape[1]):
                    if self.difference and j>= int(sample_x.shape[1]/2):
                        [cwtmatr, frequencies] = pywt.cwt(sample_x[:,j],   scales2,  self.wavename, sampling_period = 1.0/self.freq)#连续小波变换模块
                    else:
                        [cwtmatr, frequencies] = pywt.cwt(sample_x[:,j],   scales1,  self.wavename, sampling_period = 1.0/self.freq)#连续小波变换模块
                    if resize_flag:
                        cwtmatr = resize(cwtmatr, (int(self.freq/f_scale), int(self.windowsize/l_scale)), mode = 'constant')
                    scalogram.append(cwtmatr)

                scalogram = np.stack(scalogram)

                with open(os.path.join(freq_path,"{}.pickle".format(name)), 'wb') as handle:
                    pickle.dump(scalogram, handle, protocol=pickle.HIGHEST_PROTOCOL)

            with open(os.path.join(freq_path,"freq_file_name.pickle"), 'wb') as handle:
                pickle.dump(freq_file_name, handle, protocol=pickle.HIGHEST_PROTOCOL)
            if flag == "train":
                self.train_freq_file_name = freq_file_name
            else:
                self.test_freq_file_name = freq_file_name

    def Sensor_filter_acoording_to_pos_and_type(self, select, filter, all_col_names, filtertype):
        """
        select  (list or None): What location should be chosen
        filter  (list or None): whether all sensors can be filtered
        all_col_names     (list)  : Columns currently available for Filtering
        """
        if select is not None:
            if filter is None:
                raise Exception('This dataset cannot be selected by sensor {}!'.format(filtertype))
            else:
                col_names = []
                for col in all_col_names:
                    selected = False
                    for one_select in select:
                        assert one_select in filter
                        if one_select in col:
                            selected = True
                    if selected:
                        col_names.append(col)
                return col_names
        else:
            return None

    def regroup_and_reindex_all_cols(self, df):
        columns = df.columns[:-1]
        # big gourps
        groups = {}
        for col in columns:
            index = col.split("_")[-1]
            if index in groups.keys():
                groups[index].append(col)
            else:
                groups[index] = [col]
        # sub_groups
        index = 1
        columns_mapping = {}
        columns_groups = []
        for key in groups.keys():
            cols = groups[key]
            cols_set = []
            for col in cols:
                cols_set.append(col.split("_")[0])
            cols_set = set(cols_set)

            for col_begin in cols_set:
                sub_groups= []
                for col in cols:
                    if col.split("_")[0]==col_begin:
                        columns_mapping[col] = "_".join(col.split("_")[:-1])+"_"+str(index)
                        sub_groups.append("_".join(col.split("_")[:-1])+"_"+str(index))
                index= index+1
                if col_begin in ["acc","gyro","gravacc"]:
                    columns_groups.append(sub_groups)
        columns = ["sub_id"]+[columns_mapping[col] for col in df.columns[:-1]] + ["sub"]
        return columns,columns_groups


# iterative
# n = number of nodes
# Time: O(n)
# Space: O(n)

from collections import deque

def tree_includes(root, target):
  if not root:
    return False
  queue = deque([root])

  while queue:
    current = queue.popleft()

    if current.val == target:
      return True

    if current.left:
      queue.append(current.left)

    if current.right:
      queue.append(current.right)

  return False

# recursive
# n = number of nodes
# Time: O(n)
# Space: O(n)

def tree_includes(root, target):
  if not root: return False
  if root.val == target: return True

  left = tree_includes(root.left, target)
  right = tree_includes(root.right, target)

  return left or right