#	File Created:
#	Modifications and additions to code by brooksandrew

import itertools
import networkx as nx
import matplotlib.pyplot as plt
from postman_problems.turn_graph import turn_weight_function_distance, create_turn_weight_edge_attr
from postman_problems.graph import get_shortest_paths_distances, create_complete_graph, add_augmenting_path_to_graph

def create_pos_and_add_to_graph(g_full, complete_g):
    """
    Since g_full does not contain any position data,
    uses complete_g to create a position dictionary,
    and adds this data to g_full as node attributes
    Args:
        g_full (NetworkX MultiDiGraph): reconstituted graph of required and optional edges
            generated by postman_problems.graph.create_networkx_graph_from_edgelist
        complete_g (NetworkX MultiDiGraph): complete graph with all edge and node data
            generated by initialize_rpp.InnerAndOuterToEdgeListFile
    Returns:
        g_full (NetworkX MultiDiGraph): g_full with position data added to nodes
        pos (dict): node position dictionary
    """
    pos = {}
    g_full_copy = g_full.copy()
    for k in g_full.nodes():
        exec("pos[k] = (complete_g.nodes["+k+"].get('x'), complete_g.nodes["+k+"].get('y'))")
        g_full_copy.add_node(k, x = pos[k][0], y = pos[k][1])
    return g_full_copy, pos

def visualize_g_req(g_req, pos):
    """
    Plots g_req
    Args:
        g_req (NetworkX MultiDiGraph): required edge graph
            generated by postman_problems.graph.create_required_graph
        pos (dict): node position dictionary
            generated by create_pos_and_add_to_graph
    """
    fig, ax = plt.subplots()
    el = [e for e in g_req.edges(data=True)]
    nx.draw_networkx_edges(g_req, pos, edgelist=el, ax = ax)
    ax.set_xlim(left = min([pos[node][0] for node in g_req.nodes()]), right = max([pos[node][0] for node in g_req.nodes()]))
    ax.set_ylim(bottom = min([pos[node][1] for node in g_req.nodes()]), top = max([pos[node][1] for node in g_req.nodes()]))
    print('\nReconstituted Directed Graph in Solver, with Contracted Connector Edges')
    plt.show()

def make_graph_eulerian(g_req, g_full):
    """
    Directed graphs must satisfy 2 criteria to be eulerian
        strong connectivity
        in degree equals out degree for each node
    In a directed graph, the total in degree equals the total out degree
    This function augments g_req into an eulerian graph by adding edges that
    balance in degree and out degree of each node
    Args:
        g_req (NetworkX MultiDiGraph): required edge graph
            generated by postman_problems.graph.create_required_graph
        g_full (NetworkX MultiDiGraph): reconstituted graph of required and optional edges
            generated by postman_problems.graph.create_networkx_graph_from_edgelist
            augmented by create_pos_and_add_to_graph
    """
    asymmetric_nodes = []
    asymmetric_nodes_list = []
    for n in g_req.nodes():
        if g_req.in_degree(n) != g_req.out_degree(n):
            asymmetric_nodes += [(n, {'in_degree': g_req.in_degree(n), 'out_degree': g_req.out_degree(n)})]
            asymmetric_nodes_list += [n]
    nodes_to_add_out_edges = {}
    nodes_to_add_out_edges_list = []
    nodes_to_add_in_edges = {}
    nodes_to_add_in_edges_list = []
    for n in asymmetric_nodes:
        if n[1]['in_degree'] > n[1]['out_degree']:
            nodes_to_add_out_edges[n[0]] = {'edges_to_add': n[1]['in_degree']-n[1]['out_degree']}
            nodes_to_add_out_edges_list += [n[0]]
        else:
            nodes_to_add_in_edges[n[0]] = {'edges_to_add': n[1]['out_degree']-n[1]['in_degree']}
            nodes_to_add_in_edges_list += [n[0]]
    degree_matching = []
    degree_matching_depth = 1
    while len(asymmetric_nodes_list) > 0:
        degree_matching_depth += 1
        possible_degree_matching_node_pairs = []
        for (n1, n2) in itertools.combinations(asymmetric_nodes_list, 2):
            if (n1 in nodes_to_add_out_edges_list and n2 in nodes_to_add_in_edges_list) or (n1 in nodes_to_add_in_edges_list and n2 in nodes_to_add_out_edges_list):
                possible_degree_matching_node_pairs += [(n1, n2)]

        degree_matching_pairs_shortest_paths = get_shortest_paths_distances(g_full, possible_degree_matching_node_pairs, edge_weight_name=turn_weight_function_distance)
        g_degree_matching = create_complete_graph(degree_matching_pairs_shortest_paths, flip_weights=True)
        degree_matching_generator = list(nx.algorithms.max_weight_matching(g_degree_matching, True))
        for (n1, n2) in degree_matching_generator:
            if n1 in nodes_to_add_out_edges_list:
                degree_matching += [(n1, n2)]
                nodes_to_add_out_edges[n1]['edges_to_add']-=1
                nodes_to_add_in_edges[n2]['edges_to_add']-=1
            else:
                degree_matching += [(n2, n1)]
                nodes_to_add_out_edges[n2]['edges_to_add']-=1
                nodes_to_add_in_edges[n1]['edges_to_add']-=1
        print()
        for n in nodes_to_add_out_edges:
            if nodes_to_add_out_edges[n]['edges_to_add'] == 0 and n in nodes_to_add_out_edges_list:
                nodes_to_add_out_edges_list.remove(n)
                asymmetric_nodes_list.remove(n)
        for n in nodes_to_add_in_edges:
            if nodes_to_add_in_edges[n]['edges_to_add'] == 0 and n in nodes_to_add_in_edges_list:
                nodes_to_add_in_edges_list.remove(n)
                asymmetric_nodes_list.remove(n)
    print('Degree Matching Depth = {}'.format(degree_matching_depth))
    g_aug = add_augmenting_path_to_graph(g_req, g_full, degree_matching, edge_weight_name=turn_weight_function_distance)
    return g_aug

def is_graph_eulerian(g_aug):
    if nx.is_eulerian(g_aug):
        print('\nAugmented Graph is Eulerian\n')
    else:
        print('\nAugmented Graph is NOT Eulerian\n')


import pytest
from django.urls import reverse

from cars.models import Order, OrderLine

@pytest.mark.django_db
def test_delete(user_client, color_factory, brand_factory, car_model_factory, order_line_factory, order_factory):
    color = color_factory()
    brand = brand_factory()
    model = car_model_factory(brand=brand)

    order = order_factory()
    order_line_factory(order=order, model=model, color=color)

    response = user_client.delete(reverse('orders-detail', args=[order.pk]))

    assert response.status_code == 204
    assert not Order.objects.exists()
    assert not OrderLine.objects.exists()


from django.db import models
from django.contrib.auth.signals import user_logged_in, user_logged_out
from django.dispatch import receiver

class AuditLog(models.Model):
    timestamp = models.DateTimeField(auto_now=True)
    user = models.CharField(max_length=250, null=False, blank=False)
    user_ip = models.CharField(max_length=100, null=False, blank=False)
    device_info = models.CharField(max_length=500, null=True, blank=True)
    action_name = models.CharField(max_length=50, null=False, blank=False)
    table_name = models.CharField(max_length=250, null=True, blank=True)
    task_name = models.CharField(max_length=50, null=True, blank=True)
    action_details = models.CharField(max_length=200, null=True, blank=True)
    data = models.TextField(null=True, blank=True)

    def __str__(self):
        return str(self.timestamp)+'_'+self.user

    class Meta:
        db_table = 'AUTH_LogWithAudit_AuditLog'

@receiver(user_logged_in)
def user_logged_in_callback(sender, request, user, **kwargs):
    us = user.username
    x_f = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_f:
        uip = x_f.split(',')[0]
    else:
        uip = request.META.get('REMOTE_ADDR')
    try:
        deviceinfo = request.META['HTTP_USER_AGENT']
    except:
        deviceinfo = ''
    AuditLog.objects.create(user=us, user_ip=uip, action_name="login", device_info=deviceinfo, action_details=us+" logged into the system", table_name="user")

@receiver(user_logged_out)
def user_logged_out_callback(sender, request, user, **kwargs):
    us = user.username
    x_f = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_f:
        uip = x_f.split(',')[0]
    else:
        uip = request.META.get('REMOTE_ADDR')
    try:
        deviceinfo = request.META['HTTP_USER_AGENT']
    except:
        deviceinfo = ''
    AuditLog.objects.create(user=us, user_ip=uip, device_info=deviceinfo, action_name="logout", action_details=us+" logged out from the system", table_name="user")


# -*- coding: utf-8 -*-
import logging
import phonenumbers
from datetime import datetime, timedelta
from dateutil import tz
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth import authenticate
from django.db import transaction
from django.db.models import Q, Prefetch
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone
from django.utils.http import urlquote
from rest_framework import exceptions, serializers
from rest_framework.response import Response
from rest_framework.validators import UniqueValidator

from shared_foundation.custom.drf.fields import PhoneNumberField
from shared_foundation.constants import *
from shared_foundation.custom.drf.validation import MatchingDuelFieldsValidator, EnhancedPasswordStrengthFieldValidator
from shared_foundation.models import SharedUser
# from tenant_api.serializers.associate_comment import AssociateCommentSerializer
from tenant_api.serializers.skill_set import SkillSetListCreateSerializer
from tenant_api.serializers.tag import TagListCreateSerializer
from tenant_api.serializers.insurance_requirement import InsuranceRequirementListCreateSerializer
from tenant_api.serializers.vehicle_type import VehicleTypeListCreateSerializer
from tenant_foundation.models import (
    AssociateComment,
    Associate,
    Comment,
    InsuranceRequirement,
    SkillSet,
    Organization,
    VehicleType,
    HowHearAboutUsItem,
    TaskItem
)

logger = logging.getLogger(__name__)

class TaskItemAvailableAssociateListCreateSerializer(serializers.ModelSerializer):
    full_name = serializers.SerializerMethodField()
    telephone = PhoneNumberField()
    e164_telephone = serializers.SerializerMethodField()
    pretty_tags = serializers.SerializerMethodField()

    # Meta Information.
    class Meta:
        model = Associate
        fields = (
            'id',
            'type_of',
            'full_name',
            'past_30_days_activity_sheet_count',
            'telephone',
            'e164_telephone',
            'email',
            'wsib_number',
            'hourly_salary_desired',
            'score',
            'tags',
            'pretty_tags',
        )
        extra_kwargs = {
            # "is_ok_to_email": {
            #     "error_messages": {
            #         "invalid": _("Please pick either 'Yes' or 'No' choice.")
            #     }
            # },
            # "is_ok_to_text": {
            #     "error_messages": {
            #         "invalid": _("Please pick either 'Yes' or 'No' choice.")
            #     }
            # },
            # "hourly_salary_desired": {
            #     "error_messages": {
            #         "min_value": _("Ensure this value is greater than or equal to 0."),
            #         "invalid": _("Please enter a value with no $, such as 20")
            #     }
            # }
        }

    def setup_eager_loading(cls, queryset):
        """ Perform necessary eager loading of data. """
        queryset = queryset.prefetch_related(
            'owner',
            'created_by',
            'last_modified_by',
            'tags',
            'skill_sets',
            'vehicle_types',
            'comments',
            'insurance_requirements',
        )
        return queryset

    def get_full_name(self, obj):
        try:
            return str(obj)
        except Exception as e:
            return None

    def get_e164_telephone(self, obj):
        """
        Converts the "PhoneNumber" object into a "NATIONAL" format.
        See: https://github.com/daviddrysdale/python-phonenumbers
        """
        try:
            if obj.telephone:
                return phonenumbers.format_number(obj.telephone, phonenumbers.PhoneNumberFormat.E164)
            else:
                return "-"
        except Exception as e:
            return None

    def get_pretty_tags(self, obj):
        try:
            s = TagListCreateSerializer(obj.tags.all(), many=True)
            return s.data
        except Exception as e:
            return None

    @transaction.atomic
    def create(self, validated_data):
        """
        Override the `create` function to add extra functinality:
        """
        return validated_data


import ast
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import dash_table_experiments as dt
import io
import json
from lmfit.model import load_modelresult
from lmfit.model import save_modelresult
import numpy as np
import os
import pandas as pd
import plotly
import urllib
import urllib.parse

from diffcapanalyzer.databasewrappers import get_filename_pref
from diffcapanalyzer.databasewrappers import if_file_exists_in_db
from diffcapanalyzer.databasewrappers import process_data
from diffcapanalyzer.databasewrappers import macc_chardis
from diffcapanalyzer.databasefuncs import init_master_table
from diffcapanalyzer.databasefuncs import get_file_from_database
from diffcapanalyzer.chachifuncs import col_variables
from diffcapanalyzer.descriptors import generate_model

def parse_contents(
        decoded,
        filename,
        datatype,
        database,
        windowlength=9,
        polyorder=3):
    """Checks if the uploaded file exists in the database yet. Will
    process and add that file to the database if it doesn't appear in
    the master table yet. Otherwise will return html.Div that the
    file already exists in the database. """

    cleanset_name = get_filename_pref(filename) + 'CleanSet'
    # this gets rid of any filepath in the filename and just leaves the
    # clean set name as it appears in the database check to see if the
    # database exists, and if it does, check if the file exists.
    ans_p = if_file_exists_in_db(database, filename)
    if ans_p:
        df_clean = get_file_from_database(cleanset_name, database)
        new_peak_thresh = 0.7
        feedback = generate_model(
            df_clean, filename, new_peak_thresh, database)
        return 'That file exists in the database: ' + \
            str(get_filename_pref(filename))
    else:
        try:
            decoded_dataframe = decoded_to_dataframe(
                decoded, datatype, filename)
            process_data(
                filename,
                database,
                decoded_dataframe,
                datatype,
                windowlength,
                polyorder)
            df_clean = get_file_from_database(cleanset_name, database)
            new_peak_thresh = 0.7
            feedback = generate_model(
                df_clean, filename, new_peak_thresh, database)
            return 'New file has been processed: ' + \
                str(get_filename_pref(filename))
        except Exception as e:
            return 'There was a problem uploading that file. ' + \
                'Check the format of the upload file is as expected.' + \
                str(e)

def decoded_to_dataframe(decoded, datatype, file_name):
    """Decodes the contents uploaded via the app. Returns
    contents as a dataframe. Accepts only csv files for
    both Arbin and MACCOR type data."""
    if decoded is None:
        try:
            data1 = pd.read_csv(file_name, index_col=False)
        except BaseException:
            data1 = pd.read_csv(file_name, delimiter='\t', index_col=False)
    else:
        try:
            contents = io.StringIO(decoded.decode('utf-8'))
        except BaseException:
            contents = io.StringIO(decoded.decode('utf-16'))
        try:
            data1 = pd.read_csv(contents, index_col=False)
        except BaseException:
            data1 = pd.read_csv(contents, delimiter='\t', index_col=False)
    if datatype == 'ARBIN':
        data1['datatype'] = 'ARBIN'
        expected_cols = [
            'Cycle_Index',
            'Voltage(V)',
            'Current(A)',
            'Charge_Capacity(Ah)',
            'Discharge_Capacity(Ah)',
            'Step_Index']
        assert all(item in list(data1.columns) for item in expected_cols)
    elif datatype == 'MACCOR':
        data1['MaccCharLab'] = data1.apply(
            lambda row: macc_chardis(row), axis=1)
        data1['Current(A)'] = data1['Current [A]'] * data1['MaccCharLab']
        data1['datatype'] = 'MACCOR'
        expected_cols = [
            'Cycle C',
            'Voltage [V]',
            'Current [A]',
            'Cap. [Ah]',
            'Md']
        assert all(item in list(data1.columns) for item in expected_cols)
        data1.rename(
            columns={
                'Cycle C': 'Cycle_Index',
                'Voltage [V]': 'Voltage(V)',
                'Current [A]': 'Abs_Current(A)',
                'Cap. [Ah]': 'Cap(Ah)'},
            inplace=True)
    else:
        None
    return data1

def pop_with_db(filename, database):
    """Returns dataframes that can be used to populate the app graphs.
    Finds the already existing file in the database and returns
    the cleaned version (as a dataframe) and the raw version
    (also as a dataframe)."""
    cleanset_name = get_filename_pref(filename) + 'CleanSet'
    rawset_name = get_filename_pref(filename) + 'Raw'
    if if_file_exists_in_db(database, filename):
        # then the file exists in the database and we can just read it
        df_clean = get_file_from_database(cleanset_name, database)
        df_raw = get_file_from_database(rawset_name, database)
        datatype = df_clean['datatype'].iloc[0]
        (cycle_ind_col, data_point_col, volt_col, curr_col, dis_cap_col,
         char_cap_col, charge_or_discharge) = col_variables(datatype)

    else:
        df_clean = None
        df_raw = None
        peakloc_dict = {}

    return df_clean, df_raw


import cv2 as cv
import numpy as np

img = cv.imread('../photos/big-bang.jpg')
cv.imshow('Bang',img)

blank = np.zeros(img.shape[:2], dtype='uint8')

mask = cv.circle(blank, (img.shape[1]//2,img.shape[0]//2),100,255,-1)
cv.imshow('Mask',mask)

masked = cv.bitwise_and(img,img,mask=mask)
cv.imshow('Masked image',masked)

cv.waitKey(0)


import logging
import os
import shutil
from abc import ABC, abstractmethod

from pyalfe.interfaces.greedy import Greedy
from pyalfe.tools import GREEDY_PATH

try:
    import ants
except ImportError:
    pass

class ImageRegistration(ABC):
    @abstractmethod
    def register_rigid(self, fixed, moving, transform_output, init_transform):
        pass

    @abstractmethod
    def register_affine(
        self, fixed, moving, transform_output, init_transform, fast=False
    ):
        pass

    @abstractmethod
    def register_deformable(
        self, fixed, moving, transform_output, affine_transform=None
    ):
        pass

    @abstractmethod
    def reslice(self, fixed, moving, registration_output, *transform):
        pass

class GreedyRegistration(ImageRegistration):
    logger = logging.getLogger('GreedyRegistration')

    def __init__(self, greedy_path=GREEDY_PATH, threads=16):
        self.greedy_path = greedy_path
        self.threads = threads

    def reslice(self, fixed, moving, registration_output, *transform):
        cmd = Greedy(self.greedy_path)
        cmd = cmd.dim(3).threads(self.threads)
        cmd = cmd.reslice(*transform).reference(fixed)
        cmd = cmd.input_output(moving, registration_output)
        cmd.run()

    def _register_affine(
        self, dof, fixed, moving, transform_output, init_transform, fast
    ):
        cmd = Greedy(self.greedy_path)
        if 'WNCC' in cmd.run():
            metric = 'WNCC'
        else:
            metric = 'NCC'
            self.logger.warning(
                'Your version of greedy does not support weigthed normalized'
                ' cross-correlation. Falling back to simple normalized'
                ' cross-correlation. Consider upgrading greedy to the'
                ' latest version.'
            )
        if init_transform:
            cmd.initialize_affine(init_transform)
        cmd = cmd.threads(self.threads).dim(3).affine().dof(dof)
        cmd = cmd.metric(metric, 2)
        if fast:
            cmd = cmd.num_iter(100, 50, 0)
        else:
            cmd = cmd.num_iter(200, 100, 50)
        cmd = cmd.image_centers().input(fixed, moving).out(transform_output)
        cmd.run()
        return transform_output

    def register_rigid(self, fixed, moving, transform_output, init_transform=None):
        return self._register_affine(
            6, fixed, moving, transform_output, init_transform, fast=False
        )

    def register_affine(
        self, fixed, moving, transform_output, init_transform=None, fast=True
    ):
        return self._register_affine(
            12, fixed, moving, transform_output, init_transform, fast
        )

    def register_deformable(
        self,
        fixed,
        moving,
        transform_output,
        affine_transform=None,
    ):
        if not affine_transform:
            fixed_name = os.path.basename(fixed).split('.')[0]
            moving_name = os.path.basename(moving).split('.')[0]
            affine_transform = os.path.join(
                os.path.dirname(transform_output), f'{moving_name}_to_{fixed_name}.mat'
            )
            self.register_affine(fixed, moving, affine_transform, fast=False)
        if not os.path.exists(affine_transform):
            self.register_affine(fixed, moving, affine_transform, fast=False)

        cmd = Greedy(self.greedy_path)
        cmd = cmd.dim(3).threads(self.threads).metric('NCC', 2)
        cmd = cmd.epsilon(0.5).num_iter(100, 50, 10)
        if affine_transform:
            cmd = cmd.transforms(affine_transform)
        cmd.input(fixed, moving).out(transform_output)
        cmd.run()

class AntsRegistration(ImageRegistration):
    def reslice(self, fixed, moving, registration_output, *transform):
        fixed_image = ants.image_read(fixed)
        moving_image = ants.image_read(moving)
        output = ants.apply_transforms(fixed_image, moving_image, transform)
        ants.image_write(output, registration_output)

    def _register_affine(
        self, type_of_transform, fixed, moving, transform_output, init_transform
    ):
        if not transform_output:
            fixed_name = os.path.basename(fixed).split('.')[0]
            moving_name = os.path.basename(moving).split('.')[0]
            transform_output = f'{moving_name}_to_{fixed_name}.mat'
        rigid_output = ants.registration(
            ants.image_read(fixed),
            ants.image_read(moving),
            type_of_transform=type_of_transform,
            initial_transform=init_transform,
            reg_iterations=(100, 50, 10),
            verbose=True,
            flow_sigma=1,
        )
        shutil.copy(rigid_output['fwdtransforms'][0], transform_output)

    def register_rigid(self, fixed, moving, transform_output=None, init_transform=None):
        self._register_affine(
            'DenseRigid', fixed, moving, transform_output, init_transform
        )

    def register_affine(
        self, fixed, moving, transform_output=None, init_transform=None, fast=False
    ):
        if fast:
            self._register_affine(
                'AffineFast', fixed, moving, transform_output, init_transform
            )
        else:
            self._register_affine(
                'TRSAA', fixed, moving, transform_output, init_transform
            )

    def register_deformable(
        self, fixed, moving, transform_output=None, affine_transform=None
    ):
        fixed_image = ants.image_read(fixed)
        moving_image = ants.image_read(moving)

        if not affine_transform:
            fixed_name = os.path.basename(fixed).split('.')[0]
            moving_name = os.path.basename(moving).split('.')[0]
            affine_transform = f'{moving_name}_to_{fixed_name}.mat'
            deformable_output = ants.registration(fixed_image, moving_image, 'SyN')
            shutil.copy(deformable_output['fwdtransforms'][1], affine_transform)
            shutil.copy(deformable_output['fwdtransforms'][0], transform_output)

        if not os.path.exists(affine_transform):
            deformable_output = ants.registration(fixed_image, moving_image, 'SyN')
            shutil.copy(deformable_output['fwdtransforms'][1], affine_transform)
            shutil.copy(deformable_output['fwdtransforms'][0], transform_output)
        else:
            deformable_output = ants.registration(
                fixed_image, moving_image, 'SyNOnly', initial_transform=affine_transform
            )
            shutil.copy(deformable_output['fwdtransforms'][0], transform_output)


from socket import *
from time import ctime
import os

HOST = 'localhost'
PORT = 8000
ADDRESS = (HOST, PORT)
BUFFSIZE = 240689

client = socket(AF_INET, SOCK_STREAM)
client.connect(ADDRESS)

fName = input("Enter the file name of the image, with its extension: ")
client.send(bytes(fName, 'utf-8'))

currDirectory = os.getcwd()
imgPath = currDirectory + '/images/recieved.jpg'

with open(imgPath, "wb") as image_file:
    while True:
        # Receive data from the client
        data = client.recv(1024)
        if not data:
            break
        # Write the received data to the file
        image_file.write(data)

print("File recieved, total bytes: ", os.path.getsize(imgPath))

print("File saved as recieved.jpg")

client.close()


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 14 17:04:07 2019

@author: yl918888
"""
import numpy as np
from src.optimization.parameter.mcs.jones.functions import feval
from src.optimization.parameter.mcs.gls.lssplit import lssplit
#from src.optimization.parameter.mcs.gls.lssort import lssort

#------------------------------------------------------------------------------
def lsnew(func,nloc,small,sinit,short,x,p,s,alist,flist,amin,amax,alp,abest,fmed,unitlen):

    if alist[0] <= amin:
        # leftmost point already at boundary
        leftok = 0
    elif flist[0] >= max(fmed,flist[1]):
        # bad leftmost point
        # extrapolate only if no scale known or global search
        leftok = (sinit == 1 or nloc > 1 )
    else:
        # good interior leftmost point
        leftok = 1

    if alist[s-1] >= amax:
        # rightmost point already at boundary
        rightok = 0
    elif flist[s-1] >= max(fmed, flist[s-2]):
        # bad rightmost point
        # extrapolate only if no scale known or global search
        rightok = (sinit ==1 or nloc > 1 )
    else:
        # good interior rightmost point
        rightok = 1

    # number of intervals used in extrapolation step
    if sinit == 1:
        step = s-1
    else:
        step = 1

    fac = short
    # do the step
    if leftok and ( flist[0] < flist[s-1] or (not rightok) ):
        #if prt>1, disp('extrapolate at left end point'); #
        extra = 1
        al = alist[0] - (alist[0+step] - alist[0])/small
        alp = max(amin, al)
    elif rightok:
        #if prt>1, disp('extrapolate at right end point'); #
        extra=1;
        au = alist[s-1] + (alist[s-1] - alist[s-1-step])/small
        alp = min(au,amax)
    else:
        # no extrapolation
        #if prt>1, disp('split relatively widest interval'); #
        extra = 0
        lenth = [i-j for i,j in zip(alist[1:s],alist[0:s-1])]
        dist = [max(i,j,k) for i,j,k in zip([i-abest for i in alist[1:s]], [abest-i for i in alist[0:s-1]], (unitlen*np.ones(s-1)).tolist())]
        wid = [lenth[i]/dist[i] for i in range(len(lenth))]
        i = np.argmax(wid)
        wid = max(wid)
        alp, fac = lssplit(i,alist,flist,short)

    # new function value
    falp = feval(func,x+alp*p)
    alist.append(alp)
    flist.append(falp)

    return alist,flist,alp,fac


num_matches = int(input())
counter = 0
counterA = 0
counterB = 0
counterAlice = 0
counterBob  = 0
for i in range(num_matches):
	x = input().split()
	for j in x:
		counter += 1
		if ('PR' == x[counter-1]):
			print('A')
		else:
			print('B')
	counter = 0


from communities import views
from django.conf.urls import url

urlpatterns = [
    url(r'^$', views.GroupListView.as_view(), name='list'),
    url(r'^create/$', views.GroupCreateView.as_view(), name='create'),
    url(r'^(?P<pk>\d+)/$', views.GroupDetailView.as_view(), name='detail'),
    url(r'^(?P<pk>\d+)/edit/$', views.GroupUpdateView.as_view(), name='update'),

]


import numpy as np
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture as GMM
from functools import partial

class Logs():
    cur_log_file = None
    log_time_tag = None
    log_dir = './logs'

    def set_log_file(tag, src_level= -2):
        Logs.cur_log_file = Logs.get_log_fn(tag=tag, src_level=src_level)

    def get_log_fn(tag=None, src_level=-2):
        if Logs.cur_log_file is None and tag is None:
            tag = "LOG_default"

        if tag is None:
            return Logs.cur_log_file

        import datetime, os

        Logs.log_time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")

        log_time_tag_parts = Logs.log_time_tag.split('_')
        log_start = 15 * (int(log_time_tag_parts[-1]) // 15)
        day = get_day_prefix(Logs.log_time_tag)

        log_time_tag_parts[-1] = str(log_start)

        fn = tag + '_log_' + '_'.join(log_time_tag_parts) + '.txt'
        target = os.sep.join([Logs.log_dir, day])
        if not os.path.exists(target):
            os.makedirs(target)

        log_fn = os.sep.join([Logs.log_dir, day, fn])
        return log_fn

def cur_step(st=None, store=[]):
    if st is not None:
        if not store:
            store.append(st)
        else:
            store[0] = st

    return store[0]

def log(s=None, d=None, src_level=-2):
    def col_print(d, cols=4, proc=lambda i, x, d:str(x)):
        cols_out = []
        last = len(d) - 1
        for i, val in enumerate(d):
            val = proc(i, val, d)
            cols_out.append(val)
            if i == 0 and i != last:
                continue
            if i % cols == cols - 1:
                log_file.write('\t' + '   '.join(cols_out) + '\n')
                cols_out = []
            elif i == last:
                if len(cols_out) == 1:
                    log_file.write('\t' + cols_out[0] + '\n')
                elif len(cols_out) > 1:
                    log_file.write('\t' + cols_out[0] + '\n')

    log_fn = Logs.get_log_fn()

    import inspect, os
    recs = inspect.stack()
    path = str(recs[1][0].f_code).split('"')[src_level]

    lineNum = recs[1][0].f_lineno
    where = os.path.basename(path)
    where += ":" + str(lineNum)
    if not Logs.log_time_tag:
        Logs.log_time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")

    when = Logs.log_time_tag + ": "
    when += where + "-"

    if log_fn is None:
        log_fn = Logs.get_log_fn()

    with open(log_fn, 'a+') as log_file:
        log_file.write(when + s + '\n')
        special_handling = ['m_params', 'vc_params_perf', 'train_scores', 'test_scores', 'plot_data']
        if d is None:
            pass
        elif type(d) is list or str(type(d)) == "<class 'numpy.ndarray'>":
            if len(d) <= 1:
                log_file.write(str(d))
            else:
                col_print(d)
        elif type(d) is dict:
            proc = lambda i, x, d : str(x) + ': ' + str(d[x])
            col_print(d, proc=proc)
            print(d)

    if type(d) is dict and len([x for x in d if x in special_handling]) > 0:
        if 'ds' in d:
            ds = d['ds']
        else:
            ds = 'no_ds_given'

        if 'plot' in d:
            plot_type = d['plot']
        else:
            plot_type = 'no_pt_specified'

        log_special = '.'.join(log_fn.split('.')[0:-1]) + '_' + list(d)[0] + '_' + ds + '_' + plot_type + '.csv'
        d = d[list(d)[0]]  # de-nest, get to the actual data

        if type(d) is list:
            out_cols = [i for i, x in enumerate(d)]
        elif str(type(d)) == "<class 'numpy.ndarray'>":
            out_cols = [i for i, x in enumerate(d)]
        else:
            out_cols = list(d)

        import csv
        if not os.path.exists(log_special):
            with open(log_special, 'w', newline='') as log_file:
                wr = csv.writer(log_file)
                wr.writerow(['where', 'when', 'msg'] + out_cols)

        col_2 = s  # ' ' * len(where)
        col_0 = where
        col_1 = Logs.log_time_tag
        with open(log_special, 'a+', newline='') as log_file:
            wr = csv.writer(log_file)

            if d is dict or isinstance(d, dict):
                out_vals = [d[k] for k in out_cols]
            elif type(d) is list:
                out_vals = d
            elif str(type(d)) == "<class 'numpy.ndarray'>":
                out_vals = d.tolist()
            else:
                hook = True

            wr.writerow([col_0, col_1, col_2] + out_vals)

def get_day_prefix(now_str=None):
    import datetime
    if now_str is None:
        now_str = datetime.datetime.now().strftime("%m_%d_%H_%M")

    time_tag_parts = now_str.split('_')
    return "_".join(time_tag_parts[:2])

def get_plot_fn(tag=None, root_path='./plots', ftype='.png'):
    import datetime, os
    time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")
    day = get_day_prefix(time_tag)
    target = os.sep.join([root_path, day])
    if not os.path.exists(target):
        os.makedirs(target)

    fn = tag + '_' + time_tag + ftype
    full_fn = os.sep.join([root_path, day, fn])

    no_overwrite = 0
    test_fn = full_fn
    while os.path.exists(test_fn):
        no_overwrite += 1
        pos = len(full_fn) - 1
        while pos >= 0 and full_fn[pos] != '.':
            pos += -1

        fn_parts = full_fn[0:pos]

        part1 = fn_parts + "_" + str(no_overwrite)
        test_fn = part1 + ftype

    full_fn = test_fn

    return full_fn

def get_ser_fn(tag=None, ftype='.npy'):
    return get_plot_fn(tag=tag, ftype=ftype)

def get_converge(npl, ep=.0001):
    cnv = 0
    cnv_val = npl[0]

    for i, x in enumerate(npl):
        if x > (1-ep)*cnv_val and x < (1+ep)*cnv_val:
            continue
        else:
            cnv = i
            cnv_val = npl[i]

    return cnv, cnv_val

def get_plot_xmax(x, y):
    y_max_idx = np.argmax(y)
    y_min_idx = np.argmin(y)
    conv_idx, val = get_converge(y)
    idxs = [y_max_idx, conv_idx, y_min_idx]
    which_idx = np.argmax( [x[ii] for ii in idxs] )
    target_idx = idxs[which_idx]
    # x vals should be monotonic, so included 2 extra points should not result in a sub-optimal x value
    if target_idx < len(x) - 2:
        target_idx += 2

    x_max = x[target_idx]

    return x_max

def pr_fl(f, len=8):
    return str(f)[0:len]

def get_ranges(data):
    hook = True

    for k in data:
        if len(data[k].shape) == 1:
            col2 = [[x+1] for x in range(data[k].shape[0])]
            data[k] = np.column_stack((data[k], col2))
            hook = True

    #massage = lambda x, l: x + 2 if x + 2 < len(l) else x
    y_idx_at_max = [get_plot_xmax(x=data[k][:, 1], y = data[k][:, 0]) for k in data]

    all_max_x = max(y_idx_at_max)
    all_min_x = min([np.min(data[k][:, 1]) for k in data])

    all_max_y = max([np.max(data[k][:, 0]) for k in data])
    all_min_y = min([np.min(data[k][:, 0]) for k in data])

    return all_min_x, all_max_x, all_min_y, all_max_y

BSK = 'best_state'
BFK = 'best_fitness'
BFCK = 'best_fc_curve'

def get_plot_title(pr, dm, dim):
    return f"{pr}-{dm} size:{dim}"

def make_plot(
        title=None,  data=None, fn=None, rows=1, extra_data=None,
        x_label="iterations", y_label="fitness score", trunc_converged=True,
        zero_x_bnd=False, zero_y_bnd=False, ebars=False
    ):

    plt.close()
    fig, axes = plt.subplots(rows, 1, squeeze=False, figsize=(10, 10))

    if hasattr(axes, 'shape'):
        subplot_1 = axes[0, 0]

    all_min_x, all_max_x, all_min_y, all_max_y = get_ranges(data)

    lb_factor = .95
    ub_factor = 1.05
    if zero_x_bnd:
        xp_min = min(0, lb_factor * all_min_x)
    else:
        xp_min = lb_factor * all_min_x

    if zero_y_bnd:
        yp_min = min(0, lb_factor * all_min_y)
    else:
        yp_min = lb_factor * all_min_y

    x_lb = xp_min
    x_ub = ub_factor * all_max_x
    y_lb = yp_min
    y_ub = ub_factor * all_max_y

    print(f"Plot bounds = x:{pr_fl(x_lb)},{pr_fl(x_ub)}, y:{pr_fl(y_lb)},{pr_fl(y_ub)}")
    subplot_1.set_xlim(x_lb, x_ub)
    subplot_1.set_ylim(y_lb, y_ub)
    subplot_1.set_title(title)
    subplot_1.set_xlabel(x_label)
    subplot_1.set_ylabel(y_label)

    subplot_1.grid()
    colors = ['r', 'g', 'b', 'c', 'm', 'k', 'y']
    syms = ['+', '1', '2', '3', '4']

    for i, k in enumerate(data):
        log(f"{title}raw plot {k}", {'plot_data': data[k]})

        cur_x = data[k][:, 1]
        cur_y = data[k][:, 0]
        if trunc_converged:
            y_idx_at_conv, _ = get_converge(cur_y)
            y_idx_at_conv = min(len(cur_y) - 1, y_idx_at_conv + 2)
            cur_x_data_2 = cur_x[0: y_idx_at_conv + 1]
            cur_y_data_2 = cur_y[0: y_idx_at_conv + 1]

            cur_x = cur_x_data_2
            cur_y = cur_y_data_2

        log(f"{title} processed plot {k} x:", cur_x)
        log(f"processed plot {k} y:", cur_y)

        extra_text = 'None'
        lbl = k
        if not extra_data:
            pass
        else:
            if k in extra_data:
                extra_text = extra_data[k]
                buffer = '' if extra_text[0].isspace() else ' '
                lbl = k + buffer + extra_text

        log(f"plot extra data {k}:" + extra_text)

        sylen = len(syms)
        pt = syms[i%sylen]
        clen = len(colors)
        if not ebars:
            subplot_1.plot(cur_x, cur_y, color=colors[i%clen], marker=pt, linestyle='solid', label=lbl)
        else:
            yerr = np.std(cur_y)
            subplot_1.errorbar(cur_x, cur_y, yerr=yerr, color=colors[i % clen], marker=pt, linestyle='solid', label=lbl)

    subplot_1.legend(loc="lower right")

    fig.tight_layout()
    plt.margins(10, 0)

    plt.savefig(fn)
    plt.show()

DS_1 = 'digits'
DS_2 = 'Bach'
DS_3 = 'Wine'

CODE_TEST_1 = 1
CODE_TEST_2 = 2
CODE_TEST_3 = 3

ds_names = {
    CODE_TEST_1:DS_1,
    CODE_TEST_2 : DS_2,
    CODE_TEST_3 : DS_3
}

def load_datasets(dset=1):
    from sklearn.datasets import load_digits
    from sklearn.datasets import load_wine

    from numpy import genfromtxt
    from datetime import datetime as dt

    def yn2num(val):
        assert (val[0].lower() in ['y', 'n'])
        res = 1 if val.lower()[0] == 'y' else 0
        return res

    def note2num(val):
        letter = val[0].lower()
        notes = ['c', 'd', 'e', 'f', 'g', 'a', 'b']
        index = notes.index(letter)
        res = float(index)

        if len(val) > 1:
            if val[1] == 'b':
                res -= .5
            elif val[1:3].lower() == 'sh':
                res += .5
            else:
                assert (False)

        assert (res >= 0)
        return res

    def colzero(val):
        decoded = val.decode("utf-8")
        return decoded

    loaded = []
    incl_digits = False
    incl_wine = False

    # CODE_TEST_2 for Bach
    if dset == CODE_TEST_1:
        incl_digits = True
    elif dset == CODE_TEST_3:
        incl_wine = False
    elif dset != CODE_TEST_2:
        assert(False)

    d1 = dict()
    cur_ds = d1

    if incl_digits:
        X, y = load_digits(return_X_y=True)
        cur_ds['X'] = X
        cur_ds['Y'] = y
        cur_ds['name'] = DS_1
        loaded.append(cur_ds)
        return loaded

    if incl_wine:
        X, y = load_wine(return_X_y=True)
        ds2 = dict()
        ds2['X'] = X
        ds2['Y'] = y
        ds2['name'] = DS_3
        # unq_1, cnts_1 = np.unique(ds1['Y'], return_counts=True)
        #nq_2, cnts_2 = np.unique(ds2['Y'], return_counts=True)

        #log(f'For dataset {DS_1} unique = {unq_1}, counts = {cnts_1}')
        #log(f'For dataset {DS_2} unique = {unq_2}, counts = {cnts_2}')
        loaded.append(ds2)

        return loaded

    # d1 = dict()

    converters = dict()
    for i in range(17):
        converters[i] = colzero

    try:
        temp = genfromtxt(
            'data/jsbach_chorals_harmony/jsbach_chorals_harmony.data', delimiter=',',
            converters=converters
        )
    except Exception as ex:
        debug_hook = True
        import sys
        _, _, tb = sys.exc_info()
        raise ()
        pass

    labels = []
    pieces = set()
    d3 = dict()
    cur_ds = d3
    cur_ds['X'] = np.zeros((len(temp), len(temp[0]) - 2))
    # cur_ds['Y'] = np.zeros((len(temp), 1))

    for i, row in enumerate(temp):
        for j in range(len(row) - 1):
            if j == 0:
                pieces.add(row[j])
                continue

            if j > 0:
                if j in (1, 15):
                    cur_ds['X'][i, j - 1] = float(row[j])
                elif 2 <= j <= 13:
                    cur_ds['X'][i, j - 1] = yn2num(row[j])
                elif j == 14:
                    cur_ds['X'][i, j - 1] = note2num(row[j])
                else:
                    assert (False)

        labels.append(row[-1])

    label_index = set(labels)
    label_index = list(label_index)
    label_index.sort()

    label_count_1 = dict()
    for s in label_index:
        label_count_1[s] = labels.count(s)

    from sklearn.preprocessing import LabelBinarizer
    from sklearn.preprocessing import LabelEncoder
    # lb = LabelBinarizer()
    le = LabelEncoder()
    cur_ds['Y'] = le.fit_transform(labels)
    unq_1, cnts_1 = np.unique(cur_ds['Y'], return_counts=True)
    cur_ds['name'] = DS_2
    loaded.append(cur_ds)

    return loaded

max_silh_km_bach = 1575
max_silh_km_digits = 15
max_silh_gmm_bach = 2056
max_silh_gmm_digits = 17

KMEANSK = 'kmeans'
GMMK = 'gmm'

STEP1 = 's1'
STEP2 = 's2'
STEP3 = 's3'
STEP4 = 's4'
STEP5 = 's5'

max_cluster_scores = {

    STEP1:{
        DS_1: {
            KMEANSK: {'max': max_silh_km_digits} ,
            GMMK: {'max': max_silh_gmm_digits}
        },
        DS_2: {
            KMEANSK: {'max': max_silh_km_bach},
            GMMK: {'max': max_silh_gmm_bach}
        }
    },
    STEP2:{
        DS_1: {
            KMEANSK: {'max': -1},
            GMMK: {'max': -1}
        },
        DS_2: {
            KMEANSK: {'max': -1},
            GMMK: {'max': -1}
        }
    }

}

def TEST_MODE(val=None, store=[]):
    if not store:
        if not val:
            store.append(False)
        else:
            store.append(val)

    if not val:
        pass
    else:
        store[0] = val

    return store[0]

def get_digits():
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.preprocessing import OneHotEncoder

    datasets = load_datasets(CODE_TEST_1)
    cur_ds = datasets[0]

    # Normalize feature data
    scaler = MinMaxScaler()

    X_train_scaled = scaler.fit_transform(cur_ds['X'])
    y_train = cur_ds['Y']

    # One hot encode target values
    one_hot = OneHotEncoder()
    y_train_hot = one_hot.fit_transform(y_train.reshape(-1, 1)).todense()
    if TEST_MODE():
        #last = int(.1 * len(X_train_scaled))
        last = 250
        X_train_scaled = X_train_scaled[0:last]
        y_train_hot = y_train_hot[0:last]

    return X_train_scaled, y_train_hot

def get_bach():
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.preprocessing import OneHotEncoder

    datasets = load_datasets(CODE_TEST_2)
    cur_ds = datasets[0]

    # Normalize feature data
    scaler = MinMaxScaler()

    X_train_scaled = scaler.fit_transform(cur_ds['X'])
    y_train = cur_ds['Y']

    # One hot encode target values
    one_hot = OneHotEncoder()
    y_train_hot = one_hot.fit_transform(y_train.reshape(-1, 1)).todense()

    if TEST_MODE():
        #last = int(.1 * len(X_train_scaled))
        last = 250
        X_train_scaled = X_train_scaled[0:last]
        y_train_hot = y_train_hot[0:last]

    return X_train_scaled, y_train_hot

GETK= 'get_func'
TAGK = 'tag'
GET_DIGITS ={
    GETK:get_digits
}

GET_BACH ={
    GETK:get_bach
}

datasets_info = {
    DS_1: GET_DIGITS,
    DS_2: GET_BACH
}

def make_x_points(starts=None, points=None, limits=None):
    assert (len(starts) == len(points))
    assert (len(starts) == len(limits))

    res_list = []
    regions = len(starts)
    for i, s in enumerate(starts):
        cur_start = s
        cur_points = points[i]

        #validate cur_points, make sure cur_points not too large
        if i == regions - 1:  # last
            if limits[i] is not None:
                cur_points = min(cur_points, limits[i] - cur_start + 1)
        else:
            next_start = starts[i + 1]
            temp_limit = next_start - 1
            cur_points = min(cur_points, temp_limit - cur_start + 1)

        if limits[i] is None:
            if i == regions - 1:  # last
                cur_limit = cur_start + cur_points - 1
            else:
                cur_limit = starts[i + 1] - 1
        else:
            cur_limit = limits[i]

        step = int((cur_limit - cur_start) / (cur_points - 1))
        assert (step > 0)
        cur_list = list(range(cur_start, cur_limit + 1, step))
        res_list += cur_list

    return res_list

def validate_limit_list(start, limit, instances):
    if limit is None:
        limit = [None for x in start]
        limit[-1] = instances
    elif len(limit) == 1:
        last_val = limit[0]
        last_pos = len(start) - 1
        limit = [None if i != last_pos else last_val for i in range(len(start)) ]
    elif len(start) != len(limit):
        assert(False)

    if limit[-1] is None:
        limit[-1] = instances

    return limit

def gmm_score(k, get_ds=None, plot=False, cl_tag=GMMK):
    from sklearn.metrics import silhouette_score

    X, y = get_ds[GETK]()
    tag = get_ds[TAGK]
    gmm_obj = GMM(k)
    gmm_obj.fit(X)
    label = gmm_obj.predict(X)

    if plot:
        st=cur_step()

        dr_tag = 'not_red'
        if REDK in get_ds:
            dr_tag = get_ds[REDK]

        cluster_plot(
            get_ds=get_ds, labels=label, dr_tag=dr_tag,
            cl_tag=cl_tag, fn=get_plot_fn(f'clu_plt_{cl_tag}_{dr_tag}')
        )
        fn_tag = f'{cl_tag}_{dr_tag}_{st}'

    return silhouette_score(X, label)

# def run_gmm(x_pts=None, get_ds=None):
#     import numpy as np
#     from sklearn.mixture import GaussianMixture as GMM
#
#     X, y = get_ds[GETK]()
#     tag = get_ds[TAGK]
#
#     instances = X.shape[0]
#
#     if x_pts is None:
#         cl_list = list(range(2, X.shape[0])) #all instance
#     else:
#         cl_list = x_pts
#
#     print(f"Evaluting cluster sizes ({tag}): {cl_list}")
#
#     results = np.zeros((len(cl_list), 2))
#
#     for i, k in enumerate(cl_list):
#         print(f'{k} centers')
#         sc = gmm_score(X, k)
#         results[i] = [sc, k]
#
#     return results

def save_np(a, st=None, ds_tag=None, cl_tag=None, extra=None,  fn=None):
    a_fn = fn
    if st is None:
        st = cur_step()
    if not fn:
        fn_tag = f'{st}@{str(ds_tag)}@{str(cl_tag)}@{str(extra)}'
        a_fn = get_ser_fn(fn_tag)

    np.save(a_fn, a)

def kmeans_score(k, get_ds=None, plot=False, cl_tag=KMEANSK):
    from sklearn.cluster import KMeans
    from sklearn.metrics import silhouette_score

    X, y= get_ds[GETK]()
    kMean = KMeans(n_clusters=k)
    kMean.fit(X)
    label = kMean.predict(X)

    if plot:
        st = cur_step()
        dr_tag = 'not_red'
        if REDK in get_ds:
            dr_tag = get_ds[REDK]

        cluster_plot(
            get_ds=get_ds, labels=label, dr_tag=dr_tag,
            cl_tag=cl_tag, fn=get_plot_fn(f'clu_plt_{cl_tag}_{dr_tag}')
        )

    return silhouette_score(X, label)

def incr_scoring(
        cl_list, get_ds=None, ds_tag=None,
        cl_tag=None, score_func= None, savepoint=100,
        skiplast=False, update=False, targ=None
):

    X, y = datasets_info[ds_tag][GETK]()
    all = [x for x in range(2, X.shape[0])]
    if not update:
        results = np.zeros((len(all), 2))
        visited = [False] * len(results)
    else:
        results = targ
        visited = [False] * len(results)
        for i, x in enumerate(results[:,1]):
            if int(x) > 0:
                visited[i] = True

    for i, k in enumerate(cl_list):
        if i == 0 or i % savepoint == 0:
            print(f'{k} clusters')
            if skiplast and i == len(cl_list) - 1:
                pass
            elif not visited[i]:
                sc = score_func(k, get_ds=get_ds)
                results[i] = [sc, k]
                visited[i] = True
                if update:
                    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary
                    set_np_array(
                        max_cluster_scores, ds_tag, cl_tag,
                        results, st=STEP1, custom_tag=CL_SCORING_RES
                    )
    set_np_array(max_cluster_scores, ds_tag, cl_tag, results) # will save to current step dictionary

    for i, k in enumerate(cl_list):
        print(f'{k} clusters')
        if skiplast and i == len(cl_list) - 1:
            pass
        else:
            if not visited[i]:
                sc = score_func(k, get_ds=get_ds)
                results[i] = [sc, k]
                visited[i] = True
                if update:
                    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary
                    set_np_array(
                        max_cluster_scores, ds_tag, cl_tag,
                        results, st=STEP1, custom_tag=CL_SCORING_RES
                    )
        if i > 0 and i % savepoint == 0:
            print("Savepoint")
            save_np(max_cluster_scores, ds_tag=ds_tag, cl_tag=cl_tag)

    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary

    return results

CL_SCORING_RES = 'cl-scoring'

def run_cluster_scoring(
        x_pts=None, get_ds=None, cl_tag=None,
        score_func=None, use_prev=False, update=False
):
    import numpy as np
    if use_prev and update:
        ds_tag = get_ds[TAGK]
        prev_found = check_for_previous_results(
            s1=cur_step(), ds_tag=ds_tag, cl_tag=cl_tag,
            custom_tag=CL_SCORING_RES
        )
        if prev_found is not None and not update:
            return prev_found

    X, y = get_ds[GETK]()
    ds_tag = get_ds[TAGK]

    instances = X.shape[0]
    #limit = validate_limit_list(start, limit, instances)
    if x_pts is None:
        cl_list = list(range(2, X.shape[0])) #all possible cluster sizes, 2 to instance - 1
    else:
        cl_list = x_pts

    print(f"Evaluating cluster sizes: {str(cl_list[0:4])[:-1]},... {cl_list[int(len(cl_list)/2)]},... {str(cl_list[-4:])[1:]}")
    if update:
        ds_tag = get_ds[TAGK]
        prev_found = check_for_previous_results(
            s1=cur_step(), ds_tag=ds_tag, cl_tag=cl_tag,
            custom_tag=CL_SCORING_RES
        )
        results = incr_scoring(
            cl_list, get_ds=get_ds, ds_tag=ds_tag,
            cl_tag= cl_tag, score_func=score_func, update=update, targ=prev_found)
    else:
        results = incr_scoring(
            cl_list, get_ds=get_ds, ds_tag=ds_tag,
            cl_tag=cl_tag, score_func=score_func, update=update)

    set_np_array(max_cluster_scores, ds_tag, cl_tag, results, custom_tag=CL_SCORING_RES)

    return results

def dr_rca_gp(X,y, ds_tag=None):
    import numpy as np
    from sklearn import random_projection
    X = np.random.rand(100, 10000)
    transformer = random_projection.GaussianRandomProjection()
    X_new = transformer.fit_transform(X)
    return transformer, X_new

def dr_rca_sm(X,y, ds_tag=None):
    import numpy as np
    from sklearn import random_projection
    X = np.random.rand(100, 10000)
    transformer = random_projection.SparseRandomProjection()
    X_new = transformer.fit_transform(X)
    return transformer, X_new

RCAGPK = 'rcagp'
RCASMK = 'rcasm'

def dr_ica(X,y, k=None, ds_tag=None, do_whitening=True, eff_score=False):
    from sklearn import decomposition
    from scipy.stats import norm
    from scipy.stats import kurtosis

    if k is not None:
        num_components = int(kur_results[kur_max_idx, 1])
        ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening)

        return ica, ica.fit_transform(X)

    space_dim = X.shape[1]
    kur_results = np.zeros((space_dim, 2))
    for i in range(space_dim):
        num_components = i + 1
        #if num_components == space_dim:
        #    continue  # the max number of components tends to generate errors, evaluating not useful
        if num_components == 64:
            hook = True
        print(f"Evaluating {num_components} ICA components")
        ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening,
            max_iter=800)

        try:
            ica.fit(X)
        except Exception as ex:
            hook = True
            raise

        A = ica.components_
        tally = np.zeros(A.shape[0])

        for ii, v in enumerate(ica.components_):
            one_axis = np.zeros(A.shape)
            one_axis[ii] = A[ii]
            proj = np.dot(X, one_axis.T)
            proj = proj[:, ii]
            kur_score = kurtosis(proj)
            tally[ii] = np.abs(kur_score)

        kur_results[i] = [np.min(tally), num_components]

    #kur_results = np.abs(kur_results)
    max_idx = np.argmax(kur_results[:,0])
    kur_max_idx  = max_idx
    plot_tag = 'kur'
    data = {plot_tag: kur_results}
    xdata = {plot_tag: f"max y:{pr_fl(kur_results[max_idx, 0])} x:{int(kur_results[max_idx, 1])}"}
    max_y = kur_results[max_idx, 0]
    max_red = kur_results[max_idx, 1]/space_dim
    max_vals = np.repeat(max_y, len(kur_results))
    std = np.std(max_vals - kur_results[:, 0])
    thresh = (max_y - std)/ max_y

    #norm_fact = max_y/max_red
    def score2(x, y,  thresh, max_store=[]):

        cur_sc_ratio = y/max_y
        cur_red_ratio = (x/space_dim)/max_red
        score2 = cur_sc_ratio/cur_red_ratio * max_y

        if not max_store:
            max_store.append(score2)

        if cur_sc_ratio > thresh and x > .4 * space_dim: # ignore larger reductions
            if score2 > max_store[0]:
                max_store[0] = score2
        else:
            score2 = .9 * max_store[0]

        return score2 # scale for graph

    #effectiveness_score
    test = score2(kur_results[max_idx, 1], kur_results[max_idx, 0], thresh)
    eff_score = [score2(r[1], r[0], thresh) for r in kur_results[0:max_idx + 1]]
    eff_score = np.array(eff_score)
    eff_score = np.column_stack((eff_score, kur_results[:,1][0:max_idx + 1]))
    data['eff_score'] = eff_score
    max_idx = np.argmax(eff_score[:, 0])
    highest_eff_score = max_idx
    xdata['eff_score'] = f"max y:{pr_fl(eff_score[max_idx, 0])} x:{int(eff_score[max_idx, 1])}"

    make_plot(
        title=f"Kurtosis Score for ICA components ({ds_tag})", data=data, x_label="ICA component",
        y_label="Kur Score", extra_data=xdata, fn=get_plot_fn(f'ICA_exp_plot_{ds_tag}'),
    )

    num_components = int(kur_results[kur_max_idx, 1])
    ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening)

    return ica, ica.fit_transform(X)

ICAK = 'ica'
DR_ICA={
    GETK:dr_ica
}

def cluster_plot(get_ds=None, threshold=6, labels=None, dr_tag=None, cl_tag=None, fn=None):
    import numpy as np
    from main import get_plot_fn
    from main import get_digits
    import pandas as pd
    import seaborn as sns
    import matplotlib.pyplot as plt
    import matplotlib
    from sklearn.feature_selection import mutual_info_classif

    X, y = get_ds[GETK]()

    if labels is None:
        labels = y

    if type(y) is list or y is None:
        hook = True

    if len(labels.shape) > 1:  # flatten one hot encoding
        labels = np.array([np.argmax(x) for x in labels])

    try:
        res = mutual_info_classif(X, labels)
    except Exception as ex:
        hook = True
        raise

    columns = [str(x) for x in range(X.shape[1] + 1)]

    kept={}
    for score, f_name in sorted(zip(res, columns), reverse=True):
        print(f_name, score)
        kept[f_name]=score

    n1 = np.column_stack((X, labels))

    df = pd.DataFrame(n1, columns=columns)

    for c in columns[:-1]:
        if c not in list(kept)[:threshold]:
            df = df.drop(c, 1)

    matplotlib.rcParams['font.size'] = 18
    sns.set_context('talk', font_scale=1.2);
    ds_tag = get_ds[TAGK]

    dr_text = f'reduced by {dr_tag}'
    if dr_tag is None:
        dr_text = 'unreduced'
    if cl_tag is None:
        cl_text = "orig. labels"
    else:
        cl_text = f'{cl_tag} labels'

    title = f'{ds_tag}-{cl_text}, {dr_text}'

    try:
        pp=sns.pairplot(df, hue=columns[-1], corner=True, diag_kind='hist')
        pp.fig.suptitle(title)

    except Exception as ex:
        hook = True
        raise

    st = cur_step()
    if fn is None:
        fn = get_plot_fn(tag=f"reduced_{st}_{ds_tag}_{dr_tag}_{cl_tag}")

    plt.savefig(fn)

def reduced_plot(x_red):
    df1_filtered = df1[filter]
    matplotlib.rcParams['font.size'] = 18
    sns.set_context('talk', font_scale=1.2);

    plt.figure(figsize=(20, 20))
    sns.pairplot(df1_filtered, hue='Chord_label', corner=True)
    # plt.margins(10,10)

    plt.savefig(get_plot_fn(tag="bach_ds"))

def dr_pca(X,y, ds_tag=None):
    from sklearn import decomposition
    pca = decomposition.PCA(n_components=.95)
    pca.fit(X)

    x_red = pca.transform(X)
    #reduced_plot(x_red)

    return pca, x_red

DR_PCA={
    GETK:dr_pca
}

def ica_eval_plot(estimator=None, fn=None, ds_tag=None):
    import matplotlib.pyplot as plt
    from scipy.stats import kurtosis

    ica = estimator
    # for i, v in enumerate(ica.components_):

def rca_eval_plot(estimator=None, fn=None, ds_tag=None):
    pass

def pca_eval_plot(estimator=None, fn=None, ds_tag=None):
    import matplotlib.pyplot as plt
    pca = estimator
    plt.rcParams["figure.figsize"] = (12, 6)

    fig, ax = plt.subplots()

    y = np.cumsum(pca.explained_variance_ratio_)
    xi = np.arange(1, y.shape[0] + 1, step = 1)

    plt.ylim(0.0, 1.1)
    plt.plot(xi, y, marker='o', linestyle='--', color='b')

    plt.xlabel('Number of Components')
    plt.xticks(np.arange(0, 11, step=1))  # change from 0-based array index to 1-based human-readable label
    plt.ylabel('Cumulative variance (%)')
    cutoff = [(ii, v) for ii, v in enumerate(y) if v >= .95]
    last = cutoff[0][0]
    plt.title(f'The number of components needed to explain variance, cutoff component = {last}')

    plt.axhline(y=0.95, color='r', linestyle='-')
    plt.text(0.5, 0.85, '95% cut-off threshold', color='red', fontsize=16)

    ax.grid(axis='x')
    plt.savefig(fn)
    plt.show()

PCAK = 'pca'

EVALPLOTK = 'evp'
DRFUNCK = 'drfunc'
REDK = 'redmeth'

DR_TAGS = [PCAK, ICAK, RCAGPK, RCASMK]
dim_reduce_info = {
    PCAK: {DRFUNCK:dr_pca, EVALPLOTK:pca_eval_plot},
    ICAK: {DRFUNCK:dr_ica, EVALPLOTK:ica_eval_plot},
    RCAGPK: {DRFUNCK: dr_rca_gp, EVALPLOTK:rca_eval_plot},
    RCASMK: {DRFUNCK: dr_rca_sm, EVALPLOTK:rca_eval_plot}
}

def run_reduction_2(reduce_func=None, plot_func=None, get_ds=None, fn=None):
    import numpy as np
    from sklearn.cluster import KMeans
    from sklearn.metrics import silhouette_score

    X, y = get_ds[GETK]()
    tag = get_ds[TAGK]

    instances = X.shape[0]
    drfunc = reduce_func[DRFUNCK]
    estimator, reduced_x = drfunc(X, y, ds_tag=tag)
    #eval plots
    plot_func(estimator=estimator, fn=fn, ds_tag=tag)

    return reduced_x

STEP_RES = "_STEP_RESULTS"

def format_ranges(ranges):
    starts = []
    limits = []
    points = []
    for t in ranges:
        st = t[0]
        lim = t[1]
        starts.append(st)
        limits.append(lim)
        if len(t) > 2:
            pts = t[2]
        else:
            pts = 10

        points.append(pts)

    return starts, points, limits

def set_series(
    data, xdata, ptag=None,
    st=None, ds_tag=None, cl_tag=None, pts=None
):
    cur_s = get_np_array(max_cluster_scores, ds_tag, cl_tag, st=st)

    if cur_s is None:
        assert(False)
    else:
        max_idx = np.argmax(cur_s[:, 0])
        max_x = cur_s[max_idx,1]

    if pts is not None:
        assert (max_x in pts)
        fb = [int(x) in pts for x in cur_s[:, 1]]
        cur_s = cur_s[fb]
        cur_st = cur_step()

        set_np_array(max_cluster_scores, ds_tag, cl_tag, cur_s, custom_tag='trunc_plot_series', st=st)

    xdata[ptag] = f"max y:{pr_fl(cur_s[max_idx, 0])} x:{int(cur_s[max_idx, 1])}"
    data[ptag] = cur_s

def check_for_previous_results(s1=None, ds_tag=None, custom_tag=None, cl_tag=None):

    r = get_np_array(
        max_cluster_scores, ds_tag, cl_tag, custom_tag=custom_tag, st=s1
    )

    return r

def run_km_clustering(title=None, ranges=None, get_ds=None, cl_tag=KMEANSK, update=False):

    #starts, points, limits = format_ranges(ranges)
    results = run_cluster_scoring(
        x_pts=ranges, get_ds=get_ds, cl_tag=cl_tag,
        score_func=kmeans_score, update=update
    )
    ds_tag = get_ds[TAGK]
    data = {}
    xdata = {}

    st = cur_step()
    if st == STEP2:
        ptag = st + ' (reduced)'
        set_series(data, xdata, ptag=ptag, st=STEP2, ds_tag=ds_tag, cl_tag=cl_tag)
        set_series(data, xdata, ptag=STEP1, st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag)
    elif st == STEP1:
        ptag = st
        set_series(data, xdata, ptag=ptag, st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag)

    if ranges is None:
        range_text = "fullr"
    else:
        start = ranges[0]
        lim = ranges[-1]
        range_text = f'{start}-{lim}'

    ds_txt = ds_tag
    if REDK in get_ds:
        ds_txt = f'{ds_txt}_{get_ds[REDK]}'

    make_plot(
        title=title, fn=get_plot_fn(f'{cl_tag}_{ds_txt}_{st}_{range_text}'),
        data=data,
        x_label='k', y_label='Silh Scor', extra_data=xdata
    )

    return results

def run_gmm_clustering(title=None, get_ds=None, ranges=None, cl_tag=GMMK, update=False):

    #starts, points, limits = format_ranges(ranges)
    results = run_cluster_scoring(
        x_pts=ranges, get_ds=get_ds, cl_tag=cl_tag,
        score_func=gmm_score, update=update
    )

    st = cur_step()
    ds_tag = get_ds[TAGK]
    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)
    title = f'{ds_tag} {cl_tag} Silhouette Scores'

    data = {}
    xdata = {}

    pt_list = [int(x) for x in results[:, 1]]

    if st == STEP2:
        ptag = st + ' (reduced)'
        set_series(data, xdata, ptag=ptag, st=STEP2, ds_tag=ds_tag, cl_tag=cl_tag)
        set_series(
            data, xdata, ptag=STEP1,
            st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag
        )
    elif st == STEP1:
        ptag = st
        set_series(
            data, xdata, ptag=ptag,
            st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag,
            pts=pt_list  #use the same x pts for both series
        )

    if ranges is None:
        range_text = "fullr"
    else:
        start = ranges[0]
        lim = ranges[0]
        range_text = f'{start}-{lim}'

    ds_txt = ds_tag
    if REDK in get_ds:
        ds_txt = f'{ds_txt}_{get_ds[REDK]}'

    make_plot(
        title=title, fn=get_plot_fn(f'{cl_tag}_{ds_txt}_{st}_{range_text}'),
        data=data,
        x_label='k', y_label='Silh Scor', extra_data=xdata
    )
    return results

SCOREK = 'score_func'
KM_CLUSTERING = {
    GETK: run_km_clustering,
    SCOREK: kmeans_score
}
GMM_CLUSTERING = {
    GETK: run_gmm_clustering,
    SCOREK: gmm_score
}

CLUSTERING_TAGS = [KMEANSK, GMMK]

clustering_info = {
    KMEANSK: KM_CLUSTERING,
    GMMK: GMM_CLUSTERING
}

def get_step1_range(ds_tag=None, cluster_tag=None):

    try:
        if STEP_RES in max_cluster_scores[STEP1][ds_tag][cluster_tag]:
            max_idx = max_cluster_scores[STEP1][ds_tag][cluster_tag][STEP_RES]['max_idx']
            step1_series = get_np_array(max_cluster_scores, ds_tag, cluster_tag, st=STEP1)
            max_x = step1_series[max_idx, 1]
        else:
            max_x = max_cluster_scores[STEP1][ds_tag][cluster_tag]['max']
            max_x = int(max_x)

    except Exception as ex:
        hook = True
        raise

    #max_y = step1_series[max_idx:0]

    start = 2
    if max_x < 20:
        last_pt = 20
    elif max_x < 40:
        last_pt = int(2.0 * max_x)
    else:
        last_pt = int(1.2 * max_x)

    pts = last_pt - start + 1
    if last_pt - start <= 40:
        ret_rng = list(range(2, last_pt + 1))
    elif last_pt - start <= 60:
        ret_rng = list(range(2, last_pt + 1, 2))
    else:
        ret_rng = make_x_points(starts=[start], limits=[last_pt], points=[20])

    ret_rng = list(set(ret_rng).union({max_x}))
    ret_rng.sort()
    assert(max_x in ret_rng)
    max_idx = ret_rng.index(max_x)

    return ret_rng, max_idx

def get_data_stub(d=None, y=None):
    return d, y

def run_reduction(
    get_ds=None, cluster_info=None, dr_func=None,
    plot_func=None
):
    ds_tag = get_ds[TAGK]
    dr_tag = dr_func[TAGK]
    ds_x_reduced = run_reduction_2(
        reduce_func=dr_func, plot_func=plot_func, get_ds=get_ds,
        fn=get_plot_fn(f'Step_2_PCA_{ds_tag}')
    )

    save_np(
        ds_x_reduced, ds_tag=ds_tag, cl_tag="na",
        extra=f'{ds_tag}_reduced_by_{dr_tag}'
    )

    # cluster plot (baseline) reduced  dataset against labels
    cluster_plot(
        get_ds=get_ds, dr_tag=dr_tag, cl_tag='baseline',
        fn=None
    )

    get_reduced = partial(get_data_stub, d=ds_x_reduced)
    GetReduced = {
        TAGK: get_ds[TAGK], GETK: get_reduced, REDK: dr_func[TAGK]
    }
    cl_tag = cluster_info[TAGK]
    cluster_func = cluster_info[GETK]
    ranges, s1_max = get_step1_range(ds_tag=ds_tag, cluster_tag=cl_tag)
    title = f'{ds_tag} {cl_tag} Silhouette Scores\nred. by {dr_tag} dim={ds_x_reduced.shape[1]}, s1 xmax = {s1_max}'
    cl_results = cluster_func(title=title, ranges=ranges, get_ds=GetReduced)
    max_idx = np.argmax(cl_results[:,0])
    max_x = cl_results[max_idx, 1]
    score_func = cluster_info[SCOREK]
    score_func(int(max_x), get_ds=GetReduced, plot=True, cl_tag=cl_tag) #
    #gmm_score

def set_np_array(info, k1, k2, a, custom_tag=None, st=None, sfn=None): # ds = k1, k2 = CL METHOD
    import os
    if st is None:
        st = cur_step()

    if k1 not in info[st]:
        info[st][k1] = {}

    if k2 not in info[st][k1]:
        info[st][k1][k2] = {}

    if STEP_RES not in info[st][k1][k2]:
        info[st][k1][k2][STEP_RES] = {}

    if custom_tag is not None:
        if custom_tag not in info[st][k1][k2]:
            info[st][k1][k2][custom_tag] = {}

        cur_d = info[st][k1][k2][custom_tag]
    else:
        cur_d = info[st][k1][k2][STEP_RES]

    if 'a' not in cur_d:
        old_a = None
    else:
        old_a = np.array(cur_d['a'])

    if old_a is None or len(old_a) != len(a):
        ba = False
    else:
        ba = old_a == a
        ba = ba.all()

    update = False
    if 'serialized' in cur_d:
        a_fn = cur_d['serialized']
    else:
        update = True
        if sfn is not None:
            a_fn = sfn
        else:
            a_fn = get_ser_fn(tag=f'saved_npa_{st}_{k1}_{k2}')

    if not ba:
        update = True

    if update:
        max_idx = np.argmax(a[:,0])
        cur_d['max_idx']= int(max_idx)
        cur_d['a'] = a.tolist()
        np.save(a_fn, a)
        if 'png' in a_fn:
            hook = True
        cur_d['serialized'] = a_fn

    save_cluster_scores(info)
    return a

def get_np_array(info, k1, k2, custom_tag=0, st=None):
    if st is None:
        st = cur_step()

    if k1 not in info[st] or k2 not in info[st][k1]:
        hook = True

    assert (k1 in info[st] and k2 in info[st][k1])
    if not custom_tag:
        # assert(STEP_RES in info[st][k1][k2])
        cur_k = STEP_RES
    else:
        #assert (custom_tag in info[st][k1][k2])
        cur_k = custom_tag

    if cur_k not in info[st][k1][k2]:
        return None

    cur_d = info[st][k1][k2][cur_k]
    a = None
    if 'serialized' in cur_d:
        a_fn = cur_d["serialized"]
        if 'png' in a_fn:
            hook = True
        a = np.load(a_fn)
        #assert ((np.array(cur_d['a']) == a).all())
        cur_d['a'] = a.tolist()
    else:
        print(f'serialized key not found in target node: {cur_d}')

    return a

def save_cluster_scores(scores, fn = './max_cluster_scores.json'):
    import json

    with open(fn, 'w') as jfp:
        json.dump(scores, jfp)

def read_cluster_scores(scores, fn='./max_cluster_scores.json'):
    import json
    with open(fn, 'r') as jfp:
        scores_pickled = json.load(jfp)

    src = scores_pickled
    targ = scores
    for k0 in [STEP1, STEP2]:
        keys = list(src[k0])
        for k1 in keys:
            for k2 in src[k0][k1]:
                cur_d = src[k0][k1][k2]
                assert(type(cur_d) is dict)
                if type(cur_d) is dict:
                    for cur_k in cur_d:
                        if type(cur_d[cur_k]) is dict and 'serialized' in cur_d[cur_k]:
                            a_fn = cur_d[STEP_RES]['serialized']
                            if STEP_RES == cur_k:
                                read_a = get_np_array(src, k1, k2, st=k0)
                            else:
                                read_a = get_np_array(src, k1, k2, custom_tag=cur_k, st=k0)

                            cur_d[cur_k]['safe_a'] = read_a.tolist()

                cur_targ = targ[k0][k1][k2]
                for k3 in cur_d:
                    new = cur_d[k3]
                    if k3 in cur_targ:
                        old = targ[k0][k1][k2][k3]
                    else:
                        old = None
                        cur_targ[k3] = None

                    if old != new:
                        print(f'updating {k0}-{k1}-{k2}-{k3}: {old} => {new}')
                        cur_targ[k3] = new
                    else:
                        print(f'{k0}-{k1}-{k2}-{k3}: original == new')

def set_serial_store(dump_dir, match_str='saved_npa', fn='./max_cluster_scores.json'):
    import json
    import os
    flist = os.listdir(dump_dir)
    with open(fn, 'r') as jfp:
        scores_pickled = json.load(jfp)
        found = [x for x in flist if match_str in x]

        st = ds_tag = None
        for f in found:
            toks = f.split('_')
            if '@' in toks[0]:
                continue
            else:
                hook = True
                for i, tok in enumerate(toks):
                    if i == 2 and tok in [STEP1, STEP2]:
                        st = tok
                    if i == 3 and tok in [DS_1, DS_2]:
                        ds_tag = tok
                    if i == 4 and tok in [KMEANSK, GMMK]:
                        cl_tag = tok

            fp = os.path.join(dump_dir, f)
            a = np.load(fp)
            set_np_array(scores_pickled, ds_tag, cl_tag, a, st=st, sfn=fp)

    save_cluster_scores(scores_pickled)

def main(args):
    cur_arg = 'update_stored'
    if cur_arg in args:
        pos = args.index(cur_arg)
        if pos <= len(args) - 2:
            set_serial_store(args[pos + 1])

    #def cluster_plot(get_ds=None, threshold=6, labels=None, dr_tag=None, cl_tag=None, fn=None):
    if 'cluster_plots' in args:
        cur_step(STEP3)
        for ds in [DS_1, DS_2]:
            ds_info = datasets_info[ds]
            ds_info[TAGK] = ds
            ds_tag= ds_info[TAGK]
            print('Dataset native cluster plot')
            cluster_plot(get_ds=ds_info, fn=get_plot_fn(f'{ds}_against_labels'))
            print('Dataset kmeans cluster plot')
            k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
            kmeans_score(k, get_ds=ds_info, plot=True)
            print('Dataset gmm cluster plot')
            k = max_cluster_scores[STEP1][ds][GMMK]['max']
            gmm_score(k, get_ds=ds_info, plot=True)

            for dr_tag in DR_TAGS:
                if dr_tag in args:
                    print(f"Cluster plots on data reduced by {dr_tag}")
                    dr_info = dim_reduce_info[dr_tag]
                    dr_info[TAGK] = dr_tag

                    X, y = ds_info[GETK]()
                    if dr_tag == ICAK:
                        k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
                        estimator, x_red = dr_info[DRFUNCK](X, y)
                    else:
                        estimator, x_red = dr_info[DRFUNCK](X, y)

                    save_np(
                        x_red, ds_tag=ds_tag, cl_tag="na",
                        extra=f'ds_reduced_by_{dr_tag}'
                    )

                    get_reduced = partial(get_data_stub, d=x_red, y=y)
                    GetReduced = {
                        TAGK: ds_info[TAGK], GETK: get_reduced, REDK: dr_info[TAGK]
                    }

                    cluster_plot(
                        get_ds=GetReduced, fn=get_plot_fn(f'{ds}_{dr_tag}_against_labels')
                    )
                    k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
                    kmeans_score(k, get_ds=GetReduced, plot=True)
                    k = max_cluster_scores[STEP1][ds][GMMK]['max']
                    gmm_score(k, get_ds=GetReduced, plot=True)

    if 'test' in args:
        TEST_MODE(val=True)

    if 'reload' in args:
        read_cluster_scores(max_cluster_scores)

    if 'INIT' in args:
        get_ds = datasets_info[DS_2]
        get_ds[TAGK] = GMMK

        ds = get_ds[GETK]
        X, y = ds()
        lazy_search(X, y_func=gmm_score)

    if STEP1 in args or 'populate' in args:
        cur_step(st=STEP1)
        for cl_tag in CLUSTERING_TAGS:
            if cl_tag in args:
                Logs.set_log_file(f"{cl_tag}_Step_1")
                cluster_tag = cl_tag
                for arg in [DS_1, DS_2]:
                    if arg in args:
                        print(f"{cluster_tag} {arg} clustering")

                        get_ds = datasets_info[arg]
                        get_ds[TAGK] = arg

                        cl_info = clustering_info[cluster_tag]
                        cl_info[TAGK] = cluster_tag

                        if 'populate' in args:
                            X, y = get_ds[GETK]()
                            lim = X.shape[0]
                            start=2

                            rng_list=[]
                            rng_list.append(list(range(2,20)))
                            rng_list.append(list(range(2, lim, 500)))
                            rng_list.append(list(range(2, lim, 250)))
                            rng_list.append(list(range(2, lim, 50)))

                            for i, r in enumerate(rng_list):
                                if i == 0:
                                    results = cl_info[GETK](ranges=r, get_ds=get_ds)
                                    set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)
                                    set_np_array(
                                        max_cluster_scores, arg, cluster_tag,
                                        results, st=STEP1, custom_tag=CL_SCORING_RES
                                    )
                                else:
                                    results = cl_info[GETK](ranges=r, get_ds=get_ds, update=True)
                                    set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)
                                    set_np_array(
                                        max_cluster_scores, arg, cluster_tag,
                                        results, st=STEP1, custom_tag=CL_SCORING_RES
                                    )
                        else:
                            results = cl_info[GETK](ranges=None, get_ds=get_ds)

                        set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)

            save_cluster_scores(max_cluster_scores)

    if STEP2 in args:
        cur_step(st=STEP2)
        scores_recorded = True
        use_original = True
        for k1 in max_cluster_scores[STEP1]:
            if use_original:
                pass
                scores_recorded = True
            else:
                k2_keys = list(max_cluster_scores[STEP1][k1])
                filtered = [x for x in k2_keys if 'serialized' in x]
                if not filtered:
                    scores_recorded = False
                    break

        if not scores_recorded:
            read_cluster_scores(max_cluster_scores)

        for dr_tag in DR_TAGS:
            Logs.set_log_file(f"Step_2_{dr_tag}")
            if dr_tag in args:
                dr_func = dim_reduce_info[dr_tag]
                dr_func[TAGK] = dr_tag

                plot_func = dr_func[EVALPLOTK]

                for cluster_tag in [KMEANSK, GMMK]:
                    if cluster_tag in args:
                        cluster_info = clustering_info[cluster_tag]
                        cluster_info[TAGK] = cluster_tag

                        for ds_tag in [DS_1, DS_2]:
                            if ds_tag in args:
                                ds_info = datasets_info[ds_tag]
                                ds_info[TAGK] = ds_tag
                                print(f'STEP2 {dr_tag} {cluster_tag} {ds_tag}')
                                run_reduction(
                                    get_ds=ds_info, cluster_info=cluster_info, dr_func=dr_func,
                                    plot_func=plot_func
                                )

def lazy_search(s, y_func=None, stop=10):
    def valid(x):
        return x >= 0 and x <= len(s) - 1

    def hillclimb(x, y_func, s, step=1, forward=None):
        if not s_visited[x]:
            s_map[x, 1] = s_pts[x]
            print(f'calculating y at {x}')
            s_map[x, 0] = y_func(s, s_pts[x])
            cur_val = s_map[x, 0]
            s_visited[x] = True
        else:
            cur_val = s_map[x, 0]

        if forward is None:
            n_stops = [x, x]
            step = 1
            neighbors = [x - step, x + step]
            for i, v in enumerate(neighbors):
                n_stops[i] = None
                if valid(v):
                    forward = True if i == 1 else False
                    ret = hillclimb(v, y_func, s, step=step * 2, forward=forward)
                    assert(type(ret) is not list)
                    n_stops[i] = ret
            return n_stops[0], n_stops[1]
        else:
            prev_step = step/2
            prev_pos = int(x + prev_step) if not forward else int(x - prev_step)
            prev_val = s_vals[prev_pos]
            assert(s_visited[prev_pos])

            neighbor = x + step if forward else x - step

            if valid(neighbor):
                stop = x
                if max_mode:
                    if cur_val >= prev_val :
                        stop = hillclimb(neighbor, y_func, s, step=step * 2, forward=forward)
                else:
                    if cur_val < prev_val:
                        stop = hillclimb(neighbor, y_func, s, step=step * 2, forward=forward)

                return stop
            else:
                return x

    def adjust(b, f, store, pos):
        tr = store[pos]
        if bs != fs:
            if not bs:
                pass
            else:
                tr.append(bs)
            if not fs:
                pass
            else:
                tr.append(fs)

            tr = list(set(tr))
            tr.sort()
            store[pos] = tr

    trails = [[], []]

    s_map = np.zeros((s.shape[0], 2))
    s_vals = s_map[:, 0] # reversed x, y order
    for i in range(len(s)):
        s_map[i, 1] = i + 2
    s_pts = [int(x) for x in s_map[:, 1]]
    s_visited = np.repeat(False, len(s))

    starts = {0: 1, 1: len(s.shape)//2, 2: len(s) - 2}
    max_mode = True
    step = 1
    for cycle in range(stop+1):
        print(f'cycle {cycle}')
        for i, a in enumerate(trails):
            if not a:
                a.append(starts[i])

            cur_start = a[0]
            cur_end = a[-1]
            int_len = len(a)
            print(f'starting search at {cur_start}')
            bs, fs = hillclimb(cur_start, y_func, s)
            adjust(bs, fs, trails, i)

            if  int_len > 1:
                print(f'starting search at {cur_end}')
                bs, fs = hillclimb(cur_end, y_func, s)
                adjust(bs, fs, trails, i)

                print(f'endpoints = {bs, fs}')

        max_mode = not max_mode
        make_plot(data={'lazy':s_map}, title=f"Lazy search, cycle {cycle}",
                  fn=get_plot_fn(f"Lz_srch_cy_{cycle}")
        )

if __name__ == '__main__':
    import sys
    import datetime

    main(sys.argv[1:])

    time_now = datetime.datetime.now().strftime("%m/%d %H:%M")
    print(f"Finished execution at {time_now}")
    log("Finished execution")

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


class Personaje:

    def __init__(self,id,name,status,species,ttype,gender,originName,orginUrl,locationName,locationUrl,image,episode,url,created):
        self._id = id
        self._nameCh = name
        self._statusCh = status
        self._speciesCh = species
        self._typeCh = ttype
        self._genderCh = gender
        self._origin_nameCh = originName
        self._origin_urlCh = orginUrl
        self._location_NameCh = locationName
        self._location_UrlCh = locationUrl
        self._imageCh = image
        self._episodeCh = episode
        self._urlCh = url
        self._createdCh = created

    def __str__(self):
	    return '''id: {}\nName: {}\nStatus: {} \nSpecies: {}\nType: {}\nGender: {}\nOriginName: {}\nOriginUrl: {}\nLocationName: {}\nLocationUrl: {}\nImage: {}\nEpisode: {}\nUrl: {}\nCreate: {}'''.format(self._id, self._nameCh, self._statusCh, self._speciesCh, self._typeCh, self._genderCh, self._origin_nameCh, self._origin_urlCh, self._location_NameCh, self._location_UrlCh, self._imageCh, self._episodeCh, self._urlCh, self._createdCh)


import socket

# 模拟客户端的函数
def clicentFunc():
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    text = "I love jingjing"
    # 发送的数据必须是bytes格式
    data = text.encode()

    # 发送
    sock.sendto(data, ("127.0.0.1", 7852))

    receive_data, receive_addr = sock.recvfrom(200)

    receive_data = receive_data.decode()

    print(receive_data)

if __name__ == '__main__':
    clicentFunc()


from __future__ import print_function, division
import os
import torch
import pandas as pd
from skimage import io, transform
import numpy as np
import skimage
import glob
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

import torch.utils.data
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
# import torchvision.transforms as transforms
from sklearn.model_selection import train_test_split
# from torchsummary import summary
S_PATH = '/home/shasvatmukes/project/audio_classification/All_Spectrograms/Mel_Spectrograms/Recording_'

def fit(train_loader, val_loader, model, loss_fn, optimizer, n_epochs, cuda, train_mode):

    if train_mode:
        train(train_loader, model, loss_fn, optimizer, n_epochs, cuda)
    else:
        test(val_loader, model, cuda)

def train(train_loader, model, loss_fn, optimizer, num_epochs, cuda):

    total_step = len(train_loader)
    for epoch in range(num_epochs):
        for i, (images, labels) in enumerate(train_loader):
            images = np.swapaxes(images, 1, -1)
            images = images.float()
            labels = labels.long()
            #print(epoch,i)
            #print(labels)
            if cuda:
                images = images.cuda()  # to(device)
                labels = labels.cuda()  # to(device)
            #print(labels)
            outputs = model(images)
            loss = loss_fn(outputs, labels)
            # Backward and optimize
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (i + 1) % 1 == 0:
                print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' .format(
                    epoch + 1, num_epochs, i + 1, total_step, loss.item()))

def test(test_loader, model, cuda):
    model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
    with torch.no_grad():
        correct = 0
        total = 0
        for images, labels in test_loader:
            images = np.swapaxes(images, 1, -1)
            images = images.float()
            # labels = labels.float()
            # print(images.shape, 'IMAGES')
            # print(labels.shape,'LABELS')
            if cuda:
                images = images.cuda()  # to(device)
                labels = labels.cuda()  # to(device)

            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

        print('Test Accuracy of the model: {} %'.format(100 * correct / total))


import heapq as hq

def solution(scoville, K):
    answer = 0
    hq.heapify(scoville)
    while scoville[0] < K:
        first = hq.heappop(scoville)
        if not len(scoville):
            return -1
        second = hq.heappop(scoville)
        hq.heappush(scoville, first + second * 2)
        answer += 1
    return answer

scoville = [1,2,3,9,10,12]
K = 7
print(solution(scoville,K))

# import heapq
#
# def solution(scoville, K):
#     answer = 0
#     h = []
#
#     for i in scoville:
#         heapq.heappush(h, i)
#
#     # 가장 작은 수가 K보다 작으면 수행
#     while h[0] < K:
#         one = heapq.heappop(h)
#         two = heapq.heappop(h) * 2
#         mix = one + two
#         heapq.heappush(h, mix)
#         answer += 1
#
#         # 만약 힙의 길이가 1이고 가장 작은 수가 K보다 작으면
#         # 더이상 스코빌 지수를 K이상으로 만들 수 없으므로
#         if len(h) == 1 and h[0] < K:
#             return -1
#     return answer
#
#
# ''' 다른 사람 풀이 (좀 더 깔끔한 풀이!)
# import heapq as hq
#
# def solution(scoville, K):
#
#     hq.heapify(scoville)
#     answer = 0
#     while True:
#         first = hq.heappop(scoville)
#         if first >= K:
#             break
#         if len(scoville) == 0:
#             return -1
#         second = hq.heappop(scoville)
#         hq.heappush(scoville, first + second*2)
#         answer += 1
#
#     return answer
# '''
#
# ''' 210313 풀이
# import heapq as hq
#
# def solution(scoville, K):
#     h = []
#     answer = 0
#     for value in scoville:
#         hq.heappush(h, value)
#
#     while True:
#         first = hq.heappop(h)
#         if first >= K:
#             return answer
#         if not len(h):
#             return -1
#         second = hq.heappop(h)
#         mixed = first + second * 2
#         hq.heappush(h, mixed)
#         answer += 1
# '''


#nuevo comentario

import time
import sys
sys.setrecursionlimit(1000)

def factorial_r(n):
    if n == 0 or n == 1:
        return 1
    return n * factorial_r(n-1)

def factorial_for(n):
    if n == 0 or n == 1:
        return 1
    acumulador = 1.0
    for i in range(1,n+1):
        acumulador *= i
    return acumulador

def run():
    beginning = time.time()
    factorial_r(10000)
    step_1 = time.time()
    factorial_for(1000)
    step_2 = time.time()
    point_1 = step_1 - beginning
    print(point_1)

if __name__ == "__main__":

    run()


from __future__ import division
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from pygam import LinearGAM, s, f

def get_pred_label(pred):
    '''
    This functions return splitting labels
    '''
    result = []
    for i in pred:
        if i < 0 :
            result.append(1)
        else:
            result.append(0)
    return result

def split_patient(patient_ids, df):
  '''
  This function returns splitting using ids
  '''
  X_train, X_test, y_train, y_test = train_test_split(patient_ids, patient_ids, test_size=0.25)
  df_train = df[df.PERSON_CD.isin(X_train)]
  df_test = df[df.PERSON_CD.isin(X_test)]
  return df_train, df_test

def split_naive(df):
  X_train, X_test, y_train, y_test = train_test_split(df, df, test_size=0.25)
  return X_train, X_test, y_train, y_test

def get_label(df):
    '''
    This functions return splitting labels
    '''
    result = []
    for index, rows in df.iterrows():
        if rows['DIFF_HRZ'] < 0 :
            result.append(1)
        else:
            result.append(0)
    return result

#Loading Datasets
df = pd.read_csv('/Users/lin/Desktop/thesis_code/df_age.csv')

# #Splitting DataSet using ids
# X_train_id, X_test_id = split_patient(df.PERSON_CD.unique(), df = df)
# y_train_id, y_test_id = get_label(X_train_id), get_label(X_test_id)

#Splitting DataSet naively
X_train, X_test, _, _ = split_naive(df)
y_train, y_test = X_train['DIFF_HRZ'], X_test['DIFF_HRZ']

##Linear mixed effect
def Linearmixed(model, df_train, df_test):
  md = smf.mixedlm(model, data = df_train, groups=df_train["AgeGroup"])
  mdf = md.fit()
  print(mdf.summary())
  test_result = mdf.predict(df_test)
  result_pred = []
  for i in test_result:
    if i < 0 :
      result_pred.append(1)
    else:
      result_pred.append(0)

  return result_pred

#Params
numeric_columns = ['AGE', 'DOSE_MG_PER_KG', 'DOPAMINE_RATE_MEAN',
                   'HYDROMORPHONE_RATE_MEAN', 'MEDIAN_HR_5_Z_y']
categorical_columns = ['opioid_inf',
                       'sed_inf',
                       'opioid_bolus',
                       'sed_bolus',
                       'vasodi_inf',
                       'vasodi_bolus',
                       'ino_inf',
                       'ino_bolus',
                       'AgeGroup']

categorical_columns_linear = ['opioid_inf',
                       'sed_inf',
                       'opioid_bolus',
                       'sed_bolus',
                       'vasodi_inf',
                       'vasodi_bolus',
                       'ino_inf',
                       'ino_bolus',
                       'AgeGroup']
columns  = categorical_columns + numeric_columns

column_model = " + ".join(columns)
model = "DIFF_HRZ ~ " + column_model

prediction_Linear_mixed = Linearmixed(model, X_train, X_test)

##Random Forest
clf = RandomForestClassifier(max_depth=100,
                             bootstrap=True,
                             max_features = 3,
                             min_samples_leaf = 5,
                             min_samples_split = 12,
                             n_estimators = 300)
clf.fit(X_train[columns],get_label(X_train))
random_Forest_result = clf.predict(X_test[columns])

#GAM
formula = f(0)
for i,name in enumerate(categorical_columns[1:]):
    formula += f(i)
for j,name in enumerate(numeric_columns):
    formula += s(j+len(categorical_columns))
print(formula)

listings_gam = LinearGAM(formula).fit(X_train[columns], y_train)

label_true = get_label(X_test)
preds = listings_gam.predict(X_test[columns])
pred_label = get_pred_label(preds)
pred_label_mixedLinear = get_pred_label(prediction_Linear_mixed)

print("The accuracy score for GAM model is {:0.3f}".format(accuracy_score(label_true, pred_label)))
print("The accuracy score for Random Forest is: {:0.3f}".format(accuracy_score(label_true,random_Forest_result)))
print("The accuracy score for linear Mixed effect model is: {:0.3f}".format(accuracy_score(label_true,pred_label_mixedLinear)))

final_pred=[]
for i in range (len(label_true)):
  if pred_label[i] + random_Forest_result[i] + pred_label_mixedLinear[i] >= 2:
    final_pred.append(1)
  else:
    final_pred.append(0)

print("The accuracy score for final is {:0.3f}".format(accuracy_score(label_true, final_pred)))

df_result = pd.DataFrame(data = {'PERSON_CD':X_test['PERSON_CD'],'Time':X_test['DT_UTC'], 'RandomForest':random_Forest_result, 'LinearMixed':prediction_Linear_mixed, 'y_true':y_test})
df_result.to_csv('model_result.csv', index = False)

from IPython import embed
embed()