import pandas as pd

from sklearn.model_selection import train_test_split

import tensorflow as tf

import matplotlib.pyplot as plt
df = pd.read_csv("/kaggle/input/dont-overfit-ii/train.csv", index_col="id")
df.info()
X = df.drop(columns=["target"])

y = df.filter(["target"])

X_train, X_test, y_train, y_test = train_test_split(

    X.values, y.values, test_size=0.2, random_state=42

)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
input_layer = tf.keras.Input(shape=(300,))

hidden_layer_1 = tf.keras.layers.Dense(1024, activation='relu')(input_layer)

hidden_layer_2 = tf.keras.layers.Dense(512, activation='relu')(hidden_layer_1)

hidden_layer_3 = tf.keras.layers.Dense(256, activation='relu')(hidden_layer_2)

output_layer = tf.keras.layers.Dense(1, activation='sigmoid')(hidden_layer_3)

model = tf.keras.Model(inputs=input_layer, outputs=output_layer)
model.summary()
model.compile(

    optimizer='adam',

    loss='binary_crossentropy',

    metrics=['accuracy']

)

history = model.fit(

    X_train, y_train,

    batch_size=64,

    epochs=100,

    validation_split=0.25

)
plt.plot(history.history['loss'])

plt.plot(history.history['val_loss'])

plt.title('Loss')

plt.legend(['Train', 'Validation'], loc='upper left')

plt.show()

plt.plot(history.history['accuracy'])

plt.plot(history.history['val_accuracy'])

plt.title('Accuracy')

plt.legend(['Train', 'Validation'], loc='upper left')

plt.show()
input_layer = tf.keras.Input(shape=(300,))

hidden_layer_1 = tf.keras.layers.Dense(32, activation='relu')(input_layer)  # Simplifying model: less layers, less neurons

hidden_layer_1 = tf.keras.layers.Dropout(rate=0.2)(hidden_layer_1)  # Adding dropout layers

hidden_layer_2 = tf.keras.layers.Dense(16, activation='relu')(hidden_layer_1)  # Simplifying model: less layers, less neurons

hidden_layer_2 = tf.keras.layers.Dropout(rate=0.2)(hidden_layer_2)  # Adding dropout layers

output_layer = tf.keras.layers.Dense(1, activation='sigmoid')(hidden_layer_2)

model = tf.keras.Model(inputs=input_layer, outputs=output_layer)

model.compile(

    optimizer='adam',

    loss='binary_crossentropy',

    metrics=['accuracy']

)

es_callback = tf.keras.callbacks.EarlyStopping(

    monitor='val_loss',

    patience=3

)

history = model.fit(

    X_train, y_train,

    batch_size=64,

    epochs=100,

    callbacks=[es_callback],  # Stop training process earlier

    validation_split=0.25

)
plt.plot(history.history['loss'])

plt.plot(history.history['val_loss'])

plt.title('Loss')

plt.legend(['Train', 'Validation'], loc='upper left')

plt.show()

plt.plot(history.history['accuracy'])

plt.plot(history.history['val_accuracy'])

plt.title('Accuracy')

plt.legend(['Train', 'Validation'], loc='upper left')

plt.show()


from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.views.generic.base import TemplateView

urlpatterns = patterns('',
    # Examples:
    url(r'^$', TemplateView.as_view(template_name="home.html"), name='home'),
    url(r'^auth-test', 'nepal.views.auth_test', name='auth-test'),
    url(r'^twitter/', include('twython_auth.urls')),
    url(r'^api/', include('api.urls', namespace='api')),
    url(r'^admin/', include(admin.site.urls)),
)

urlpatterns += patterns('',
    (r'^django-rq/', include('django_rq.urls')),
)


import inspect

from tasks.core import main, pool
from tasks.driver import COMMANDS

# --------------- Задание 1.1 --------------- #

with open("config.json", encoding="utf-8") as utu:
    l = json.loads(utu.read())
utu.close()
DISPLAY = l.get('display').get('show')
DISPLAY_MARGIN = l.get('display').get('margin')
PREFIX = l.get('prefix')

if name == 'main':
    main.configure(
        display=DISPLAY,
        display_margin=DISPLAY_MARGIN,
        base_prefix=PREFIX,
        commands=COMMANDS
    )
    while True:
        main.execute(pool.next())


import numpy as np
import argparse
import cv2

ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image",
    help="path to the (optional) video file")
args = vars(ap.parse_args())

def nothing(x):
    pass

cap = cv2.VideoCapture(0)
cv2.namedWindow("Trackbars")

cv2.createTrackbar("L - H", "Trackbars", 0, 179, nothing)
cv2.createTrackbar("L - S", "Trackbars", 0, 255, nothing)
cv2.createTrackbar("L - V", "Trackbars", 0, 255, nothing)
cv2.createTrackbar("U - H", "Trackbars", 179, 179, nothing)
cv2.createTrackbar("U - S", "Trackbars", 255, 255, nothing)
cv2.createTrackbar("U - V", "Trackbars", 255, 255, nothing)

while True:

    frame = cv2.imread(args["image"])
    height, width, channels = frame.shape

    frame = cv2.resize(frame,(int(width/3), int(height/3)))

    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    l_h = cv2.getTrackbarPos("L - H", "Trackbars")
    l_s = cv2.getTrackbarPos("L - S", "Trackbars")
    l_v = cv2.getTrackbarPos("L - V", "Trackbars")
    u_h = cv2.getTrackbarPos("U - H", "Trackbars")
    u_s = cv2.getTrackbarPos("U - S", "Trackbars")
    u_v = cv2.getTrackbarPos("U - V", "Trackbars")

    lower_blue = np.array([l_h, l_s, l_v])
    upper_blue = np.array([u_h, u_s, u_v])
    mask = cv2.inRange(hsv, lower_blue, upper_blue)

    result = cv2.bitwise_and(frame, frame, mask=mask)

    cv2.imshow("frame", frame)
    cv2.imshow("mask", mask)
    cv2.imshow("result", result)

    key = cv2.waitKey(1)
    if key == 27:
        break

cap.release()
cv2.destroyAllWindows()


from airflow.models import Connection
from airflow.upgrade.rules.base_rule import BaseRule
from airflow.utils.db import provide_session

class SparkJDBCOperatorConnIdRule(BaseRule):
    title = "Check Spark JDBC Operator default connection name"

    description = """\
In Airflow 1.10.x, the default value for SparkJDBCOperator class 'conn_id' is 'spark-default'.
From Airflow 2.0, it has been changed to 'spark_default' to conform with the naming conventions
of all other connection names.
    """

    @provide_session
    def check(self, session=None):
        for conn in session.query(Connection.conn_id):
            if conn.conn_id == 'spark-default':
                return (
                    "Deprecation Warning: From Airflow 2.0, the default value of 'conn_id' argument of "
                    "SparkJDBCOperator class has been changed to 'spark_default' to conform with the naming "
                    "conventions of all other connection names. Please rename the connection with "
                    "id 'spark-default' to 'spark_default' or explicitly pass 'spark-default' "
                    "to the operator. See the link below for details: "
                    "https://github.com/apache/airflow/blob/2.0.0/"
                    "UPDATING.md#sparkjdbchook-default-connection"
                )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('core', '0004_card'),
    ]

    operations = [
        migrations.CreateModel(
            name='VoucherUseLog',
            fields=[
                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
                ('date_spent', models.DateTimeField(auto_now_add=True)),
                ('comment', models.CharField(max_length=100)),
                ('vouchers', models.DecimalField(decimal_places=0, max_digits=3)),
            ],
        ),
        migrations.CreateModel(
            name='VoucherWallet',
            fields=[
                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
                ('cached_balance', models.DecimalField(decimal_places=2, max_digits=8, editable=False, default=0)),
                ('semester', models.ForeignKey(to='core.Semester', on_delete=models.CASCADE)),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
            ],
        ),
        migrations.CreateModel(
            name='WorkLog',
            fields=[
                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
                ('date_issued', models.DateTimeField(auto_now_add=True)),
                ('date_worked', models.DateField()),
                ('work_group', models.CharField(max_length=20)),
                ('hours', models.DecimalField(decimal_places=1, max_digits=8)),
                ('vouchers', models.DecimalField(decimal_places=2, max_digits=8)),
                ('comment', models.CharField(max_length=100)),
                ('issuing_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
                ('wallet', models.ForeignKey(to='voucher.VoucherWallet', on_delete=models.CASCADE)),
            ],
        ),
        migrations.AddField(
            model_name='voucheruselog',
            name='wallet',
            field=models.ForeignKey(to='voucher.VoucherWallet', on_delete=models.CASCADE),
        ),
        migrations.AlterUniqueTogether(
            name='voucherwallet',
            unique_together=set([('user', 'semester')]),
        ),
    ]


import logging

from plone.autoform.form import AutoExtensibleForm
from Products.CMFCore.utils import getToolByName
from Products.CMFCore.WorkflowCore import WorkflowException
from z3c.form import form
from z3c.form import button
from z3c.form.interfaces import HIDDEN_MODE
from zope import component
from zope import interface
from zope import schema

from collective.rcse.i18n import _
from collective.rcse.page.controller.users_directory import UsersDirectoryView
from Products.statusmessages.interfaces import IStatusMessage

logger = logging.getLogger(__name__)

class ManageUserFormSchema(interface.Interface):
    """Form used by user to register."""
    user_id = schema.ASCIILine(title=_(u"Users uuid"))

class ManageUserFormAdapter(object):
    component.adapts(interface.Interface)
    interface.implements(ManageUserFormSchema)

    def __init__(self, context):
        self.context = context
        self.user_id = None

class ManageUserForm(AutoExtensibleForm, form.Form):
    schema = ManageUserFormSchema
    enableCSRFProtection = True

    def _handleUser(self, action):
        if self.request.response.status in (301, 302):
            return
        data, errors = self.extractData()
        message = IStatusMessage(self.request)
        if errors:
            message.add("%s" % errors, type="error")
            logger.info(errors)
            return
        user = self._getUserItem(data['user_id'])
        if user is None:
            logger.info("user data not found")
            message.add(_(u"user not found in directory"), type="error")
            return
        wtool = getToolByName(self.context, 'portal_workflow')
        try:
            wtool.doActionFor(user, action)
        except WorkflowException as e:
            message.add(_(u"Can't enable the member ${user}",
                          mapping={'user': user}),
                        type="error")
            logger.error("Can't enable the member %s: %s" %
                         (user.username, e.message))
            return
        referer = self.request.get("HTTP_REFERER")
        if not referer:
            referer = self.context.absolute_url()
        self.request.response.redirect(referer)

    def _getUserItem(self, id):
        portal = getToolByName(self.context, 'portal_url').getPortalObject()
        try:
            user = portal['users_directory'][id]
        except KeyError:
            user = None
        return user

class ManagePendingUserForm(ManageUserForm):

    def updateActions(self):
        super(ManageUserForm, self).updateActions()
        self.actions['approve'].addClass('btn-primary')

    @button.buttonAndHandler(_(u"Approve"), name="approve")
    def handleApprove(self, action):
        self._handleUser('approve')

    @button.buttonAndHandler(_(u"Decline"), name="decline")
    def handleDisapprove(self, action):
        self._handleUser('decline')

class ManageDisabledUserForm(ManageUserForm):

    def updateActions(self):
        super(ManageUserForm, self).updateActions()
        self.actions['enable'].addClass('btn-primary')

    @button.buttonAndHandler(_(u"Enable"), name="enable")
    def handleApprove(self, action):
        self._handleUser('enable')

    @button.buttonAndHandler(_(u"Delete"), name="delete")
    def handleDelete(self, action):
        data, errors = self.extractData()
        userid = data['user_id']
        if not userid in self.context:
            return  # User has already been deleted
        self.context.manage_delObjects([userid])
        msg = _(u"User ${user} has been deleted",
                mapping={'user': userid})
        IStatusMessage(self.request).add(msg)
        referer = self.request.get("HTTP_REFERER")
        if not referer:
            referer = self.context.absolute_url()
        self.request.response.redirect(referer)

class ManagePendingUsersView(UsersDirectoryView):
    state = "pending"

    def getUsersForms(self):
        users = self.getMembers(review_state=self.state)
        forms = []
        for user in users:
            user['form'] = ManagePendingUserForm(self.context, self.request)
            user['form'].next_url = '/@@rcse_users_manage'
            user['form'].update()
            user['form'].widgets['user_id'].mode = HIDDEN_MODE
            user['form'].widgets['user_id'].value = user["dataid"]
            forms.append(user)
        return forms

class ManageDisabledUsersView(UsersDirectoryView):
    state = "disabled"

    def getUsersForms(self):
        users = self.getMembers(review_state=self.state)
        forms = []
        for user in users:
            user['form'] = ManageDisabledUserForm(self.context, self.request)
            user['form'].next_url = '/@@rcse_users_manage'
            user['form'].update()
            user['form'].widgets['user_id'].mode = HIDDEN_MODE
            user['form'].widgets['user_id'].value = user["dataid"]
            forms.append(user)
        return forms


from __future__ import absolute_import, division, print_function

from functools import partial, wraps
from itertools import product
from math import factorial, log, ceil
import operator

import numpy as np
from toolz import compose, partition_all, merge, get

from . import chunk
from .core import _concatenate2, Array, atop, sqrt, lol_tuples
from .numpy_compat import divide
from ..compatibility import getargspec, builtins
from ..base import tokenize
from ..context import _globals
from ..utils import ignoring

def reduction(x, chunk, aggregate, axis=None, keepdims=None, dtype=None,
              split_every=None, combine=None):
    """ General version of reductions

    >>> reduction(my_array, np.sum, np.sum, axis=0, keepdims=False)  # doctest: +SKIP
    """
    if axis is None:
        axis = tuple(range(x.ndim))
    if isinstance(axis, int):
        axis = (axis,)
    axis = tuple(i if i >= 0 else x.ndim + i for i in axis)

    if dtype and 'dtype' in getargspec(chunk).args:
        chunk = partial(chunk, dtype=dtype)
    if dtype and 'dtype' in getargspec(aggregate).args:
        aggregate = partial(aggregate, dtype=dtype)

    # Normalize split_every
    split_every = split_every or _globals.get('split_every', 4)
    if isinstance(split_every, dict):
        split_every = dict((k, split_every.get(k, 2)) for k in axis)
    elif isinstance(split_every, int):
        n = builtins.max(int(split_every ** (1/(len(axis) or 1))), 2)
        split_every = dict.fromkeys(axis, n)
    else:
        split_every = dict((k, v) for (k, v) in enumerate(x.numblocks) if k in axis)

    # Map chunk across all blocks
    inds = tuple(range(x.ndim))
    tmp = atop(partial(chunk, axis=axis, keepdims=True), inds, x, inds)
    tmp._chunks = tuple((1,)*len(c) if i in axis else c for (i, c)
                        in enumerate(tmp.chunks))

    # Reduce across intermediates
    depth = 1
    for i, n in enumerate(tmp.numblocks):
        if i in split_every and split_every[i] != 1:
            depth = int(builtins.max(depth, ceil(log(n, split_every[i]))))
    func = compose(partial(combine or aggregate, axis=axis, keepdims=True),
                   partial(_concatenate2, axes=axis))
    for i in range(depth - 1):
        tmp = partial_reduce(func, tmp, split_every, True, None)
    func = compose(partial(aggregate, axis=axis, keepdims=keepdims),
                   partial(_concatenate2, axes=axis))
    return partial_reduce(func, tmp, split_every, keepdims=keepdims,
                          dtype=dtype)

def partial_reduce(func, x, split_every, keepdims=False, dtype=None, name=None):
    """Partial reduction across multiple axes.

    Parameters
    ----------
    func : function
    x : Array
    split_every : dict
        Maximum reduction block sizes in each dimension.

    Example
    -------
    Reduce across axis 0 and 2, merging a maximum of 1 block in the 0th
    dimension, and 3 blocks in the 2nd dimension:

    >>> partial_reduce(np.min, x, {0: 1, 2: 3})    # doctest: +SKIP
    """
    name = name or 'p_reduce-' + tokenize(func, x, split_every, keepdims, dtype)
    parts = [list(partition_all(split_every.get(i, 1), range(n))) for (i, n)
             in enumerate(x.numblocks)]
    keys = product(*map(range, map(len, parts)))
    out_chunks = [tuple(1 for p in partition_all(split_every[i], c)) if i
                  in split_every else c for (i, c) in enumerate(x.chunks)]
    if not keepdims:
        out_axis = [i for i in range(x.ndim) if i not in split_every]
        getter = lambda k: get(out_axis, k)
        keys = map(getter, keys)
        out_chunks = list(getter(out_chunks))
    dsk = {}
    for k, p in zip(keys, product(*parts)):
        decided = dict((i, j[0]) for (i, j) in enumerate(p) if len(j) == 1)
        dummy = dict(i for i in enumerate(p) if i[0] not in decided)
        g = lol_tuples((x.name,), range(x.ndim), decided, dummy)
        dsk[(name,) + k] = (func, g)
    return Array(merge(dsk, x.dask), name, out_chunks, dtype=dtype)

@wraps(chunk.sum)
def sum(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.empty((1,), dtype=a._dtype).sum().dtype
    else:
        dt = None
    return reduction(a, chunk.sum, chunk.sum, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

@wraps(chunk.prod)
def prod(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.empty((1,), dtype=a._dtype).prod().dtype
    else:
        dt = None
    return reduction(a, chunk.prod, chunk.prod, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

@wraps(chunk.min)
def min(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.min, chunk.min, axis=axis, keepdims=keepdims,
                     dtype=a._dtype, split_every=split_every)

@wraps(chunk.max)
def max(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.max, chunk.max, axis=axis, keepdims=keepdims,
                     dtype=a._dtype, split_every=split_every)

@wraps(chunk.any)
def any(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.any, chunk.any, axis=axis, keepdims=keepdims,
                     dtype='bool', split_every=split_every)

@wraps(chunk.all)
def all(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.all, chunk.all, axis=axis, keepdims=keepdims,
                     dtype='bool', split_every=split_every)

@wraps(chunk.nansum)
def nansum(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = chunk.nansum(np.empty((1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, chunk.nansum, chunk.sum, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

with ignoring(AttributeError):
    @wraps(chunk.nanprod)
    def nanprod(a, axis=None, dtype=None, keepdims=False, split_every=None):
        if dtype is not None:
            dt = dtype
        elif a._dtype is not None:
            dt = np.empty((1,), dtype=a._dtype).nanprod().dtype
        else:
            dt = None
        return reduction(a, chunk.nanprod, chunk.prod, axis=axis,
                         keepdims=keepdims, dtype=dt, split_every=split_every)

@wraps(chunk.nanmin)
def nanmin(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.nanmin, chunk.nanmin, axis=axis,
                     keepdims=keepdims, dtype=a._dtype, split_every=split_every)

@wraps(chunk.nanmax)
def nanmax(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.nanmax, chunk.nanmax, axis=axis,
                     keepdims=keepdims, dtype=a._dtype, split_every=split_every)

def numel(x, **kwargs):
    """ A reduction to count the number of elements """
    return chunk.sum(np.ones_like(x), **kwargs)

def nannumel(x, **kwargs):
    """ A reduction to count the number of elements """
    return chunk.sum(~np.isnan(x), **kwargs)

def mean_chunk(x, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    n = numel(x, dtype=dtype, **kwargs)
    total = sum(x, dtype=dtype, **kwargs)
    result = np.empty(shape=n.shape,
              dtype=[('total', total.dtype), ('n', n.dtype)])
    result['n'] = n
    result['total'] = total
    return result

def mean_combine(pair, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    n = sum(pair['n'], **kwargs)
    total = sum(pair['total'], **kwargs)
    result = np.empty(shape=n.shape, dtype=pair.dtype)
    result['n'] = n
    result['total'] = total
    return result

def mean_agg(pair, dtype='f8', **kwargs):
    return divide(pair['total'].sum(dtype=dtype, **kwargs),
                  pair['n'].sum(dtype=dtype, **kwargs), dtype=dtype)

@wraps(chunk.mean)
def mean(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.mean(np.empty(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, mean_chunk, mean_agg, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every, combine=mean_combine)

def nanmean(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.mean(np.empty(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(mean_chunk, sum=chunk.nansum, numel=nannumel),
                     mean_agg, axis=axis, keepdims=keepdims, dtype=dt,
                     split_every=split_every,
                     combine=partial(mean_combine, sum=chunk.nansum, numel=nannumel))

with ignoring(AttributeError):
    nanmean = wraps(chunk.nanmean)(nanmean)

def moment_chunk(A, order=2, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    total = sum(A, dtype=dtype, **kwargs)
    n = numel(A, **kwargs)
    u = total/n
    M = np.empty(shape=n.shape + (order - 1,), dtype=dtype)
    for i in range(2, order + 1):
        M[..., i - 2] = sum((A - u)**i, dtype=dtype, **kwargs)
    result = np.empty(shape=n.shape, dtype=[('total', total.dtype),
                                            ('n', n.dtype),
                                            ('M', M.dtype, (order-1,))])
    result['total'] = total
    result['n'] = n
    result['M'] = M
    return result

def _moment_helper(Ms, ns, inner_term, order, sum, kwargs):
    M = Ms[..., order - 2].sum(**kwargs) + sum(ns * inner_term**order, **kwargs)
    for k in range(1, order - 1):
        coeff = factorial(order)/(factorial(k)*factorial(order - k))
        M += coeff * sum(Ms[..., order - k - 2] * inner_term**k, **kwargs)
    return M

def moment_combine(data, order=2, ddof=0, dtype='f8', sum=np.sum, **kwargs):
    kwargs['dtype'] = dtype
    kwargs['keepdims'] = True

    totals = data['total']
    ns = data['n']
    Ms = data['M']
    total = totals.sum(**kwargs)
    n = sum(ns, **kwargs)
    mu = divide(total, n, dtype=dtype)
    inner_term = divide(totals, ns, dtype=dtype) - mu
    M = np.empty(shape=n.shape + (order - 1,), dtype=dtype)

    for o in range(2, order + 1):
        M[..., o - 2] = _moment_helper(Ms, ns, inner_term, o, sum, kwargs)

    result = np.zeros(shape=n.shape, dtype=[('total', total.dtype),
                                            ('n', n.dtype),
                                            ('M', Ms.dtype, (order-1,))])
    result['total'] = total
    result['n'] = n
    result['M'] = M
    return result

def moment_agg(data, order=2, ddof=0, dtype='f8', sum=np.sum, **kwargs):
    totals = data['total']
    ns = data['n']
    Ms = data['M']

    kwargs['dtype'] = dtype
    # To properly handle ndarrays, the original dimensions need to be kept for
    # part of the calculation.
    keepdim_kw = kwargs.copy()
    keepdim_kw['keepdims'] = True

    n = sum(ns, **keepdim_kw)
    mu = divide(totals.sum(**keepdim_kw), n, dtype=dtype)
    inner_term = divide(totals, ns, dtype=dtype) - mu

    M = _moment_helper(Ms, ns, inner_term, order, sum, kwargs)
    return divide(M, sum(n, **kwargs) - ddof, dtype=dtype)

def moment(a, order, axis=None, dtype=None, keepdims=False, ddof=0,
           split_every=None):
    if not isinstance(order, int) or order < 2:
        raise ValueError("Order must be an integer >= 2")
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(moment_chunk, order=order), partial(moment_agg,
                     order=order, ddof=ddof), axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every,
                     combine=partial(moment_combine, order=order))

@wraps(chunk.var)
def var(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, moment_chunk, partial(moment_agg, ddof=ddof), axis=axis,
                     keepdims=keepdims, dtype=dt, split_every=split_every,
                     combine=moment_combine)

def nanvar(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(moment_chunk, sum=chunk.nansum, numel=nannumel),
                     partial(moment_agg, sum=np.nansum, ddof=ddof), axis=axis,
                     keepdims=keepdims, dtype=dt, split_every=split_every,
                     combine=partial(moment_combine, sum=np.nansum))

with ignoring(AttributeError):
    nanvar = wraps(chunk.nanvar)(nanvar)

@wraps(chunk.std)
def std(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    result = sqrt(a.var(axis=axis, dtype=dtype, keepdims=keepdims, ddof=ddof,
                        split_every=split_every))
    if dtype and dtype != result.dtype:
        result = result.astype(dtype)
    return result

def nanstd(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    result = sqrt(nanvar(a, axis=axis, dtype=dtype, keepdims=keepdims,
                         ddof=ddof, split_every=split_every))
    if dtype and dtype != result.dtype:
        result = result.astype(dtype)
    return result

with ignoring(AttributeError):
    nanstd = wraps(chunk.nanstd)(nanstd)

def vnorm(a, ord=None, axis=None, dtype=None, keepdims=False, split_every=None):
    """ Vector norm

    See np.linalg.norm
    """
    if ord is None or ord == 'fro':
        ord = 2
    if ord == np.inf:
        return max(abs(a), axis=axis, keepdims=keepdims, split_every=split_every)
    elif ord == -np.inf:
        return min(abs(a), axis=axis, keepdims=keepdims, split_every=split_every)
    elif ord == 1:
        return sum(abs(a), axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)
    elif ord % 2 == 0:
        return sum(a**ord, axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)**(1./ord)
    else:
        return sum(abs(a)**ord, axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)**(1./ord)

def _arg_combine(data, axis, argfunc):
    """Merge intermediate results from ``arg_*`` functions"""
    vals = data['vals']
    arg = data['arg']
    ns = data['n']
    args = argfunc(vals, axis=axis)
    offsets = np.roll(np.cumsum(ns, axis=axis), 1, axis)
    offsets[tuple(slice(None) if i != axis else 0 for i in range(ns.ndim))] = 0
    inds = list(reversed(np.meshgrid(*map(np.arange, args.shape), sparse=True)))
    inds.insert(axis, args)

    arg = (arg + offsets)[tuple(inds)]
    vals = vals[tuple(inds)]
    n = ns.sum(axis=axis).take(0, 0)
    return arg, vals, n

def arg_chunk(func, argfunc, x, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    vals = func(x, axis=axis, keepdims=True)
    arg = argfunc(x, axis=axis, keepdims=True)
    result = np.empty(shape=vals.shape, dtype=[('vals', vals.dtype),
                                               ('arg', arg.dtype),
                                               ('n', 'i8')])
    result['vals'] = vals
    result['arg'] = arg
    result['n'] = x.shape[axis]
    return result

def arg_combine(func, argfunc, data, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    arg, vals, n = _arg_combine(data, axis, argfunc)
    shape = tuple(s if i != axis else 1 for (i, s) in enumerate(data.shape))
    result = np.empty(shape=shape, dtype=[('vals', vals.dtype),
                                          ('arg', arg.dtype),
                                          ('n', 'i8')])
    result['vals'] = vals.reshape(shape)
    result['arg'] = arg.reshape(shape)
    result['n'] = n
    return result

def arg_agg(func, argfunc, data, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    return _arg_combine(data, axis, argfunc)[0]

def arg_reduction(func, argfunc):
    chunk = partial(arg_chunk, func, argfunc)
    agg = partial(arg_agg, func, argfunc)
    combine = partial(arg_combine, func, argfunc)
    @wraps(argfunc)
    def _(a, axis=None, split_every=None):
        if axis < 0:
            axis = a.ndim + axis
        return reduction(a, chunk, agg, axis=axis, dtype='i8',
                         split_every=split_every, combine=combine)
    return _

argmin = arg_reduction(chunk.min, chunk.argmin)
argmax = arg_reduction(chunk.max, chunk.argmax)
nanargmin = arg_reduction(chunk.nanmin, chunk.nanargmin)
nanargmax = arg_reduction(chunk.nanmax, chunk.nanargmax)

def cumreduction(func, binop, ident, x, axis, dtype=None):
    """ Generic function for cumulative reduction

    Parameters
    ----------
    func: callable
        Cumulative function like np.cumsum or np.cumprod
    binop: callable
        Associated binary operator like ``np.cumsum->add`` or ``np.cumprod->mul``
    ident: Number
        Associated identity like ``np.cumsum->0`` or ``np.cumprod->1``
    x: dask Array
    axis: int
    dtype: dtype

    Returns
    -------
    dask array

    See also
    --------
    cumsum
    cumprod
    """
    if dtype is None:
        dtype = func(np.empty((0,), dtype=x.dtype)).dtype
    assert isinstance(axis, int)
    m = x.map_blocks(func, axis=axis, dtype=dtype)

    name = '%s-axis=%d-%s' % (func.__name__, axis, tokenize(x, dtype))
    n = x.numblocks[axis]
    full = slice(None, None, None)
    slc = (full,) * axis + (slice(-1, None),) + (full,) * (x.ndim - axis - 1)

    indices = list(product(*[range(nb) if i != axis else [0]
                             for i, nb in enumerate(x.numblocks)]))
    dsk = dict()
    for ind in indices:
        shape = tuple(x.chunks[i][ii] if i != axis else 1
                      for i, ii in enumerate(ind))
        dsk[(name, 'extra') + ind] = (np.full, shape, ident, m.dtype)
        dsk[(name,) + ind] = (m.name,) + ind

    for i in range(1, n):
        last_indices = indices
        indices = list(product(*[range(nb) if ii != axis else [i]
                                 for ii, nb in enumerate(x.numblocks)]))
        for old, ind in zip(last_indices, indices):
            last = (name, 'extra') + old
            this_slice = (name, 'extra') + ind
            dsk[this_slice] = (binop, (name, 'extra') + old,
                                      (operator.getitem, (m.name,) + old, slc))
            dsk[(name,) + ind] = (binop, this_slice, (m.name,) + ind)

    return Array(merge(dsk, m.dask), name, x.chunks, m.dtype)

@wraps(np.cumsum)
def cumsum(x, axis, dtype=None):
    return cumreduction(np.cumsum, operator.add, 0, x, axis, dtype)

@wraps(np.cumprod)
def cumprod(x, axis, dtype=None):
    return cumreduction(np.cumprod, operator.mul, 1, x, axis, dtype)


# coding: utf8
import tensorflow as tf

class TextCNN(object):

    def __init__(self, vocab_size, seq_len, num_classes):
        self.vocab_size = vocab_size
        self.seq_len = seq_len
        self.num_classes = num_classes

        self.embedding_dim = 64
        self.num_filters = 100
        self.kernel_sizes = [3, 4, 5]
        self.hidden_dim = 100
        self.learning_rate = 1e-3

        self.build_graph()

    def feed_dict(self, sentences, labels):
        return {self.input_sentence: sentences, self.input_label: labels}

    def build_graph(self):
        self.input_sentence = tf.placeholder(tf.int32, [None, self.seq_len], name='input_sentence')
        self.input_label = tf.placeholder(tf.int64, [None], name='input_label')

        with tf.device("/cpu:0"):
            embedding = tf.get_variable('embedding', [self.vocab_size, self.embedding_dim])
            embedding_sentence = tf.nn.embedding_lookup(embedding, self.input_sentence)

        with tf.name_scope("cnn"):
            pools = []
            for kernel_size in self.kernel_sizes:
                conv = tf.layers.conv1d(embedding_sentence, self.num_filters, kernel_size, name='conv%d' % (kernel_size))
                pool = tf.reduce_max(conv, reduction_indices=[1], name='pool%d' % (kernel_size))
                pools.append(pool)

            pool = tf.concat(pools, axis=1)

        with tf.name_scope("fc"):
            fc = tf.layers.dense(pool, self.hidden_dim, name='fc1')
            #fc = tf.contrib.layers.dropout(fc, self.keep_prob)
            fc = tf.nn.relu(fc)

            self.logits = tf.layers.dense(fc, self.num_classes, name='fc2')
            self.y_pred_cls = tf.argmax(tf.nn.softmax(self.logits), 1)

        with tf.name_scope("optimize"):
            cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.input_label)
            self.loss = tf.reduce_mean(cross_entropy)
            self.optim = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.loss)

        with tf.name_scope("accuracy"):
            correct_pred = tf.equal(self.input_label, self.y_pred_cls)
            self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))


import matplotlib.pyplot as plt
from config import config as cfg
import os

dest = os.path.dirname(__file__)

def frames_plot(mode, list_of_calls, list_of_frames, timestamp):

    # every file will be saved to given dir
    target = dest + "/" + timestamp
    os.makedirs(target, exist_ok=True)
    os.path.join(target)

    if cfg["SUB"]["USE_FIFO_TO_LRU"]:
        mode = "FIFO"

    title_text = f'{mode} TABLE FOR {cfg[mode]["FRAMES_RANGE"]} FRAMES, ' \
                 f'{cfg[mode]["CALLS_RANGE"]} CALLS'

    # size of plot
    WIDTH = cfg[mode]["CALLS_RANGE"]/6 + 4
    HEIGHT = cfg[mode]["FRAMES_RANGE"]/3
    fig, gnt = plt.subplots(figsize=(WIDTH, HEIGHT))

    # title of plot
    adder = cfg[mode]["FRAMES_RANGE"]*cfg[mode]["CALLS_RANGE"]/100
    y_value = 0.85+adder
    plt.suptitle(title_text, y=min(y_value, 0.95), size=8)

    # creating content of the table
    data = [list_of_calls]
    for frame in list_of_frames:
        data.append(frame.get_results())
    column_labels = [i for i in range(cfg[mode]["CALLS_RANGE"])]
    row_labels = ["CALLS"]
    for i in list_of_frames:
        row_labels.append(f'{str(i)}')

    # creating table continued
    gnt.axis('tight')
    gnt.axis('off')
    colColours = ["lime"] * cfg[mode]["CALLS_RANGE"]
    rowColours = ["gold"] + ["cornflowerblue"] * cfg[mode]["FRAMES_RANGE"]
    cellColours = []
    callColours = []

    # coloring calls (upper part of table)
    for i in range(cfg[mode]["CALLS_RANGE"]):
        callColours.append("yellow")
    cellColours.append(callColours)
    for frame in list_of_frames:
        cell_row = []
        for item in range(cfg[mode]["CALLS_RANGE"]):
            if item in frame.log:
                # show faults
                cell_row.append("lightsalmon")
            else:
                # show hits
                cell_row.append("lightcyan")
        cellColours.append(cell_row)

    # table finalise
    gnt.table(cellText=data, cellLoc='center', rowLabels=row_labels,
              colLabels=column_labels, cellColours=cellColours,
              rowLoc='center', loc='center', colColours=colColours,
              rowColours=rowColours)

    # save to file
    title = f"{mode}_TABLE_" + timestamp + ".png"
    sub = timestamp + "/"
    plt.savefig(sub + title)


"""Implements a price level class that is a (linked list) queue capsule and a
BST node."""

from .quote import Order

class Level:
    """Class to represent a price level in the OrderBook.

    Attributes
    ----------
    price : float
        Price of all the limit orders stored in queue
    is_bid : bool
        Indicates if the level contains buy (`True`) or sell (`False`) orders
    total_qty : int
        Total amount of shares to be traded in the level
    head : Order
        Points to the first order in queue
    tail : Order
        Points to the last order in queue
    right : Level
        Points to a right-son level in the BST (BookSide)
    left : Level
        Points to a left-son level in the BST (BookSide)
    parent : Level
        Points to its parent level in the BST (BookSide)

    Methods
    -------
    enqueue_order(limit_order: Order) -> None:
        adds a new limit order in the end of the queue

    dequeue_order() -> Order:
        removes and returns the limit order in the front of the queue

    insert_order(limit_order: Order) -> None:
        inserts a limit order considering its timestamp
        relative to the ones already in queue

    remove_order(limit_order: Order) -> None:
        removes a specific limit order inside the queue
        identifying it by its timestamp
    """

    def __init__(self, limit_order: Order) -> None:
        """Build a new price level based on a limit order.

        Args:
            limit_order (Order)
        """
        # Level data #
        self.price: float = limit_order.price
        self.is_bid: bool = limit_order.is_buy
        self.total_qty: int = limit_order.qty
        # References to the first and last nodes in the Queue #
        self.head: Order = limit_order
        self.tail: Order = limit_order
        # Binary Search Tree node fields #
        self.right: Level = None
        self.left: Level = None
        self.parent: Level = None

    def enqueue_order(self, limit_order: Order) -> None:
        """Add a new limit order in the end of the queue.

        Args:
            limit_order (Order): limit order to be added
        """
        self.total_qty += limit_order.qty
        if self.head is None:
            self.head = limit_order
        else:
            self.tail.next = limit_order
        limit_order.prev = self.tail
        self.tail = limit_order

    def dequeue_order(self) -> Order:
        """Remove the limit order in the front of the queue.

        Returns:
            Order: the first limit order in queue
        """
        order = self.head
        if order is not None:
            self.head = order.next
            if self.head is not None:
                self.head.prev = None
        self.total_qty -= order.qty
        if order == self.tail:
            self.tail = None
        return order

    def insert_order(self, limit_order: Order) -> None:
        """Insert a limit order considering its timestamp relative to the ones
        already in queue.

        Args:
            limit_order (Order): limit order to be added
        """
        if limit_order.timestamp >= self.tail.timestamp:
            self.enqueue_order(limit_order)
            return
        self.total_qty += limit_order.qty
        pointer: Order = self.head
        aux: Order = None
        while pointer.timestamp >= limit_order.timestamp:
            aux = pointer
            pointer = pointer.next
        aux.next = limit_order
        pointer.prev = limit_order
        limit_order.next = pointer
        limit_order.prev = aux

    def remove_order(self, limit_order: Order) -> None:
        """Remove a specific limit order inside the queue identifying it by its
        timestamp.

        Args:
            limit_order (Order): limit order to be removed
        """
        if limit_order == self.head:
            self.dequeue_order()
            return
        self.total_qty -= limit_order.qty
        pointer: Order = self.head
        aux: Order = None
        while pointer != limit_order:
            aux = pointer
            pointer = pointer.next
        aux.next = limit_order.next
        if limit_order.next is not None:
            limit_order.next.prev = aux


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Pattern exploration and detection

Explore and detect patterns (loops, borders, centromeres, etc.) in Hi-C contact
maps with pattern matching.

Usage:
    declooptor.py detect <contact_maps> [<output>] [--kernel=None] [--loops]
                         [--borders] [--precision=4] [--iterations=auto]
                         [--output]

Arguments:
    -h, --help                  Display this help message.
    --version                   Display the program's current version.
    contact_maps                The Hi-C contact maps to detect patterns on, in
                                CSV format. File names must be separated by a
                                colon.
    -k None, kernel None        A custom kernel template to use, if not using
                                one of the presets. If not supplied, the
                                loops or borders option must be used.
                                [default: None]
    -L, --loops                 Whether to detect chromatin loops.
    -B, --borders               Whether to detect domain borders.
    -p 4, --precision 4         Precision threshold when assessing pattern
                                probability in the contact map. A lesser value
                                leads to potentially more detections, but more
                                false positives. [default: 4]
    -i auto, --iterations auto  How many iterations to perform after the first
                                template-based pass. Auto means iterations are
                                performed until convergence. [default: auto]
    -o, --output                Output directory to write the detected pattern
                                coordinates, agglomerated plots and matrix
                                images into.

"""

import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import savgol_filter
import utils
import pathlib
import itertools
import functools
import docopt

from version import __version__

MAX_ITERATIONS = 50

def pattern_detector(
    matrices,
    kernel,
    pattern_type="loops",
    precision=4.0,
    area=8,
    undetermined_percentage=1.,
    labels=None,
):

    if isinstance(matrices, np.ndarray):
        matrix_list = [matrices]
    else:
        matrix_list = matrices

    if labels is None:
        labels = range(len(matrix_list))
    elif isinstance(labels, str):
        labels = [labels]

    pattern_windows = []  # list containing all pannel of detected patterns
    pattern_sums = np.zeros(
        (area * 2 + 1, area * 2 + 1)
    )  # sum of all detected patterns
    agglomerated_pattern = np.zeros(
        (area * 2 + 1, area * 2 + 1)
    )  # median of all detected patterns
    detected_patterns = []
    n_patterns = 0

    for matrix, name in zip(matrix_list, labels):

        detrended, threshold_vector = utils.detrend(matrix)
        matrix_indices = np.where(matrix.sum(axis=0) > threshold_vector)
        n = matrix.shape[0]

        res2 = utils.corrcoef2d(
            detrended, kernel, centered_p=False
        )  # !!  Here the pattern match  !!
        res2[np.isnan(res2)] = 0.0
        n2 = res2.shape[0]
        res_rescaled = np.zeros(np.shape(matrix))
        res_rescaled[
            np.ix_(
                range(int(area), n2 + int(area)),
                range(int(area), n2 + int(area)),
            )
        ] = res2
        VECT_VALUES = np.reshape(res_rescaled, (1, n ** 2))
        VECT_VALUES = VECT_VALUES[0]
        thr = np.median(VECT_VALUES) + precision * np.std(VECT_VALUES)
        indices_max = np.where(res_rescaled > thr)
        indices_max = np.array(indices_max)
        res_rescaled = np.triu(res_rescaled)
        res_rescaled[(res_rescaled) < 0] = 0
        pattern_peak = utils.picker(res_rescaled, thr)

        if pattern_peak != "NA":
            if pattern_type == "loops":
                # Assume all loops are not found too far-off in the matrix
                mask = (
                    np.array(abs(pattern_peak[:, 0] - pattern_peak[:, 1]))
                    < 5000
                )
                pattern_peak = pattern_peak[mask, :]
                mask = (
                    np.array(abs(pattern_peak[:, 0] - pattern_peak[:, 1])) > 2
                )
                pattern_peak = pattern_peak[mask, :]
            elif pattern_type == "borders":
                # Borders are always on the diagonal
                mask = (
                    np.array(abs(pattern_peak[:, 0] - pattern_peak[:, 1])) == 0
                )
                pattern_peak = pattern_peak[mask, :]
            for l in pattern_peak:
                if l[0] in matrix_indices[0] and l[1] in matrix_indices[0]:
                    p1 = int(l[0])
                    p2 = int(l[1])
                    if p1 > p2:
                        p22 = p2
                        p2 = p1
                        p1 = p22
                    if (
                        p1 - area >= 0
                        and p1 + area + 1 < n
                        and p2 - area >= 0
                        and p2 + area + 1 < n
                    ):
                        window = detrended[
                            np.ix_(
                                range(p1 - area, p1 + area + 1),
                                range(p2 - area, p2 + area + 1),
                            )
                        ]
                        if (
                            len(window[window == 1.])
                            < ((area * 2 + 1) ** 2)
                            * undetermined_percentage
                            / 100.
                        ):  # there should not be many indetermined bins
                            n_patterns += 1
                            score = res_rescaled[l[0], l[1]]
                            detected_patterns.append([name, l[0], l[1], score])
                            pattern_sums += window
                            pattern_windows.append(window)
                        else:
                            detected_patterns.append([name, "NA", "NA", "NA"])
        else:
            detected_patterns.append([name, "NA", "NA", "NA"])

    # Computation of stats on the whole set - Agglomerated procedure :
    for i in range(0, area * 2 + 1):
        for j in range(0, area * 2 + 1):
            list_temp = []
            for el in range(1, len(pattern_windows)):
                list_temp.append(pattern_windows[el][i, j])
            agglomerated_pattern[i, j] = np.median(list_temp)

    return detected_patterns, agglomerated_pattern

border_detector = functools.partial(
    pattern_detector, pattern_type="borders", undetermined_percentage=20.
)
loop_detector = functools.partial(
    pattern_detector, pattern_type="loops", undetermined_percentage=1.
)

PATTERN_DISPATCHER = {"loops": loop_detector, "borders": border_detector}
PRESET_KERNEL_PATH = pathlib.Path("/home/rsg/DATA/Loops_Detector/data")

def load_kernels(pattern):
    """Load pattern kernels

    Look for one or several kernel file (in CSV format).

    Parameters
    ----------
    pattern : str
        The pattern type. Must be one of 'borders', 'loops' or 'centromeres',
        but partial matching is allowed.

    Returns
    -------
    pattern_kernels : list
        A list of array_likes corresponding to the loaded patterns.
    """

    pattern_path = pathlib.Path(pattern)
    if pattern_path.is_dir():
        pattern_globbing = pattern_path.glob("*")
    elif pattern in PATTERN_DISPATCHER.keys():
        print("cas spécial")
        pattern_globbing = PRESET_KERNEL_PATH.glob("*{}*".format(pattern))
    else:
        pattern_globbing = (pattern_path,)
    for kernel_file in pattern_globbing:
        yield np.loadtxt(kernel_file)

def explore_patterns(
    matrices,
    pattern_type="loops",
    custom_kernels=None,
    precision=4,
    iterations="auto",
    window=4,
):
    """Explore patterns in a list of matrices

    Given a pattern type, attempt to detect that pattern in each matrix with
    confidence determined by the precision parameter. The detection is done
    in a multi-pass process:
    - First, pattern matching is done with the initial supplied kernels.
    - Then, an 'agglomerated' median pattern from all previously detected
    patterns is generated, and detection is done using this pattern for
    matching instead.
    - Repeat as needed or until convergence.

    Parameters
    ----------
    matrices : iterable
        A list (or similarly iterable object) of matrices to detect patterns
        on.
    pattern_type : file, str or pathlib.Path
        The type of pattern to detect. Must be one of 'borders', 'loops', or
        'centromeres', but partial matching is allowed. If it looks like a
        path, instead the file is loaded and used as a tempalte itself.
    precision : float, optional
        The confidence with which pattern attribution is performed. The lower,
        the more detected patterns, the more false positives. Default is 4.
    iterations : str or int, optional
        How many iterations after the first kernel-based pass to perform in
        order to detect more patterns. If set to 'auto', iterations are
        performed until no more patterns are detected from one pass to the
        next.
    window : int, optional
        The pattern window area. When a pattern is discovered in a previous
        pass, further detected patterns falling into that area are discarded.

    Returns
    -------
    all_patterns : dict
        A dictionary in the form 'chromosome': list_of_coordinates_and_scores,
        and it is assumed that each matrix corresponds to a different
        chromosome. The chromosome string is determined by the matrix filename.
    agglomerated_patterns : list
        A list of agglomerated patterns after each pass.
    """

    # Dispatch detectors: the border detector has specificities while the
    # loop detector is more generic, so we use the generic one by default if
    # a pattern specific detector isn't implemented.
    my_pattern_detector = PATTERN_DISPATCHER.get(pattern_type, loop_detector)

    if custom_kernels is None:
        chosen_kernels = load_kernels(pattern_type)
    else:
        chosen_kernels = load_kernels(custom_kernels)

    # Init parameters for the while loop:
    #   - There's always at least one iteration (with the kernel)
    #   - Loop stops when the same number of patterns are detected after an
    #     iterations, or the max number of iterations has been specified and
    #     reached.
    #   - After the first pass, instead of the starting kernel the
    #     'agglomerated pattern' is used for pattern matching.

    all_patterns = set()
    agglomerated_patterns = [list(chosen_kernels)]
    iteration_count = 0
    old_pattern_count, current_pattern_count = -1, 0
    # Depending on matrix resolution, a pattern may be smeared over several
    # pixels. This trimming function ensures that there won't be many patterns
    # clustering around one location.

    def clean_by_neighborhood(coord_list):
        for my_pattern in coord_list:

            chromosome, pos1, pos2, _ = my_pattern
            if pos1 != "NA":
                pos1 = int(pos1)
                pos2 = int(pos2)

                neighbours = set(
                    zip(
                        itertools.repeat(chromosome),
                        itertools.product(
                            range(pos1 - window, pos1 + window + 1),
                            range(pos2 - window, pos2 + window + 1),
                        ),
                    )
                )

                if not neighbours.intersection(all_patterns):
                    yield (chromosome, pos1, pos2)

    while old_pattern_count != current_pattern_count:
        print(old_pattern_count, current_pattern_count)

        if iteration_count >= MAX_ITERATIONS or (
            iterations != "auto" and iteration_count >= iterations
        ):
            print("J'ai fini")
            break

        agglomerated_patterns.append([])
        old_pattern_count = current_pattern_count
        iteration_count += 1
        for kernel in agglomerated_patterns[-2]:
            (detected_coords, agglomerated_pattern) = my_pattern_detector(
                matrices, kernel, precision=precision
            )
            for new_coords in clean_by_neighborhood(detected_coords):
                all_patterns.add(new_coords)
            agglomerated_patterns[-1].append(agglomerated_pattern)

        current_pattern_count = len(all_patterns)

    return all_patterns, agglomerated_patterns

def pattern_plot(patterns, matrix, name=None, output=None):
    print(patterns)
    if name is None:
        name = 0
    if output is None:
        output = pathlib.Path()

    th_sum = np.median(matrix.sum(axis=0)) - 2.0 * np.std(matrix.sum(axis=0))
    matscn = utils.scn_func(matrix, th_sum)
    plt.imshow(matscn ** 0.15, interpolation="none", cmap="afmhot_r")
    plt.title(name, fontsize=8)
    plt.colorbar()

    for pattern_type, all_patterns in patterns.items():
        if pattern_type == "borders":
            for border in all_patterns:
                if border[0] != name:
                    continue
                if border[1] != "NA":
                    _, pos1, pos2 = border
                    plt.plot(pos1, pos2, "D", color="white", markersize=.5)
        elif pattern_type == "loops":
            for loop in all_patterns:
                if loop[0] != name:
                    continue
                if loop[1] != "NA":
                    _, pos1, pos2 = loop
                    plt.scatter(
                        pos1, pos2, s=15, facecolors="none", edgecolors="gold"
                    )

    plt.savefig(
        output / (str(pathlib.Path(str(name))) + ".pdf2"),
        dpi=100,
        format="pdf",
    )
    plt.close("all")

def distance_plot(matrices, labels=None):

    if isinstance(matrices, np.ndarray):
        matrix_list = [matrices]
    else:
        matrix_list = matrices

    if labels is None:
        labels = range(len(matrix_list))
    elif isinstance(labels, str):
        labels = [labels]

    for matrix, name in zip(matrices, labels):
        dist = utils.distance_law(matrix)
        x = np.arange(0, len(dist))
        y = dist
        y[np.isnan(y)] = 0.
        y_savgol = savgol_filter(y, window_length=17, polyorder=5)
        plt.plot(x, y, "o")
        plt.plot(x)
        plt.plot(x, y_savgol)
        plt.xlabel("Genomic distance")
        plt.ylabel("Contact frequency")
        plt.xlim(10 ** 0, 10 ** 3)
        plt.ylim(10 ** -5, 10 ** -1)
        plt.loglog()
        plt.title(name)
        plt.savefig(pathlib.Path(name) / ".pdf3", dpi=100, format="pdf")
        plt.close("all")

def agglomerated_plot(agglomerated_pattern, pattern_type="patterns"):

    plt.imshow(
        agglomerated_pattern,
        interpolation="none",
        vmin=0.,
        vmax=2.,
        cmap="seismic",
    )
    plt.colorbar()
    plt.title("Agglomerated plot of detected {}".format(pattern_type))
    plt.savefig(
        "agglomerated_{}.pdf".format(pattern_type), dpi=100, format="pdf"
    )
    plt.close("all")

def main():
    arguments = docopt.docopt(__doc__, version=__version__)

    contact_maps = arguments["<contact_maps>"].split(",")
    kernel = arguments["--kernel"]
    loops = arguments["--loops"]
    borders = arguments["--borders"]
    precision = float(arguments["--precision"])
    iterations = arguments["--iterations"]
    output = arguments["<output>"]

    if not output:
        output = pathlib.Path()
    else:
        output = pathlib.Path(output)

    output.mkdir(exist_ok=True)

    try:
        iterations = int(iterations)
    except ValueError:
        if iterations != "auto":
            print('Error! Iterations must be an integer or "auto"')
            return

    patterns_to_explore = set()
    if loops:
        patterns_to_explore.add("loops")
    if borders:
        patterns_to_explore.add("borders")

    patterns_to_plot = dict()

    loaded_maps = [np.loadtxt(contact_map) for contact_map in contact_maps]
    for pattern in patterns_to_explore:
        all_patterns, agglomerated_patterns = explore_patterns(
            loaded_maps, pattern, precision=precision
        )
        patterns_to_plot[pattern] = all_patterns
    for matrix in loaded_maps:
        pattern_plot(patterns_to_plot, matrix, output=output)

if __name__ == "__main__":
    main()


import argparse

from Project import find_project
from log import log_error

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Super Simple Package Manager for C/C++")
    parser.add_argument("--clean", choices=["build", "libs", "all"], help="Completely cleans the lib directory EXCEPT lib/cache.")
    parser.add_argument("--verbose", action="store_true", help="Turn on verbose mode.")
    args = parser.parse_args()

    project = find_project(args.verbose)
    if not project:
        log_error("Project", "Unable to find project file!")
        exit(0)

    if args.clean:
        project.clean(args.clean)
        exit(0)

    project.make_directories()

    project.setup_dependencies()


class Student:

    def __init__(self, *args):
        self.name = args[0]
        self.last_name = args[1]
        self.birth_year = args[2]
        # calculate the student_id here
        self.student_id = self.name[0] + self.last_name + self.birth_year

input_data = []
for i in range(3):
    input_data.append(str(input()))

child = Student(*input_data)
print(child.student_id)


# -*- coding: utf-8 -*-
# @Time: 2020/3/31 13:24
# @Author: Rollbear
# @Filename: example_of_relatives.py
# 实例：亲属关系推理
from lib.relatives import Relatives
from entity.system import System
from entity.horn_clause import HornClause

def main():
    # 证明目标：Tom和Sally是兄妹关系（不区分年龄）
    # 注：如果更换证明目标后证明失败，先检查库中的规则是否满足证明需求
    t = HornClause(None, [Relatives.bro_sister.exec(["Tom", "Sally"])])

    # 事实集合
    facts_set = [
        # 事实：John和Tom是父子关系
        HornClause(Relatives.father_son.exec(["John", "Tom"]), []),
        # 事实：John和Sally是父女关系
        HornClause(Relatives.father_dau.exec(["John", "Sally"]), [])
    ]

    # 实例化一个证明系统
    sys = System(target=t)

    # 导入上面指定的事实集合
    sys.add_facts(facts=facts_set)

    # 从亲属关系库中导入亲属关系规则
    sys.add_rules(rules=Relatives.get_relatives_rules())

    # 打印系统中现有的所有子句并开始推理
    sys.show()
    res = sys.run(debug=True)
    if res:
        print("证明成功")
    else:
        print("证明失败")

if __name__ == '__main__':
    main()


"""
=====================
Quantum Circuit Simulator
=====================
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import plot, ion, show
from matplotlib.colors import ListedColormap
import random
from collections import Counter
import itertools
from numpy.linalg import multi_dot

"Ground state"
def ggs(num_qubits): #which is short for get_ground_state(num_qubits):
    "return vector of size 2**num_qubits with all zeroes except first element which is 1"
    ground_state = np.array([np.zeros(2**num_qubits)])
    ground_state[0][0] = 1
    return ground_state.T

"Quantum Logic Gates"

"One Qubit Gates"
X = np.array([[0, 1], [1, 0]])
Y = np.array([[0, -1j], [1j, 0]])
Z = np.array([[1, 0], [0, -1]])
H = np.dot(2**(-0.5), np.array([[1, 1],[1, -1]]))
I = np.array([[1, 0], [0, 1]])
def U(theta, nx, ny, nz):
    "theta is the rotation angle and (nx, ny, nz) should be a unit vector in Cartesian coordinates in the Bloch sphere describing the rotation axis"
    return 1j*np.cos(theta/2)*I + np.sin(theta/2)*(nx*X + ny*Y + nz*Z)

"Product Gates"
def gpg(arg): #which is short for get_product_gate(*arg):
    #input the gates that you want to apply using square brackets and quotation marks (e.g. ["X","H","Z"] if you want X to the first qubit, H to the second and Z to the final qubit)
    "return matrix of size 2**num_qubits by 2**num_qubits"
    if len(arg) == 2:
        return np.kron(eval(arg[-2]), eval(arg[-1]))
    else:
        argWithoutFirst = arg[1:]
        return np.kron(eval(arg[0]), gpg(argWithoutFirst))

"Controlled Gates"
"Define useful matrices, such as the projection operations |0><0| and |1><1|"
P0X0 = np.array([[1, 0], [0, 0]])
P1X1 = np.array([[0, 0], [0, 1]])

def gcg(*arg): #which is short for get_controlled_gate(*arg):
    #input the gates that you want to apply using quotation marks (e.g. "c","X","I","I" if you want the first to be the control and X to be (conditionally) applied to the second qubit while the third and fourth qubit is uninvolved
    "return matrix of size 2**num_qubits by 2**num_qubits"
    n = 0
    for x in arg:
        n = n + x.count('c')
    if n == 0:
        print("If there are no controls, please use gpg to implement the circuit correctly.")
    if n > 1:
        print("I cannot apply gates with multiple controls.")
    else:
        controlled_gate = np.zeros((2**len(arg),2**len(arg)))
        arg0 = ["P0X0" if x == 'c' else 'I' for x in arg]
        arg1 = ["P1X1" if x == 'c' else x for x in arg]
        controlled_gate = gpg(arg0) + gpg(arg1)
        return controlled_gate

"Compile and run the programme"
def comp(circuit): #which is short for compile
    circuit_eval = [eval(circuit[x]) for x in range(len(circuit))]
    if len(circuit_eval) == 1:
        return circuit_eval[0]
    else:
        return np.linalg.multi_dot(circuit_eval)

def run_programme(circuit, ground_state):
    final_state = np.dot(comp(circuit), ground_state)
    return final_state

"Get the counts"

"This creates a list of all the possible outputs"
def bin_list(n):
     return ["".join(seq) for seq in itertools.product("01", repeat = int(n))]

def measure_all(state_vector, num_shots):
    weights = [np.abs(state_vector.tolist()[x])**2 for x in range(len(state_vector.tolist()))]
    n = np.log2(len(state_vector.tolist()))
    outputs = random.choices(bin_list(n), weights, k = num_shots)
    cwzm = Counter(outputs) #which is short for counts_without_zeros_mentioned
    list_of_all_counts = ['%s : %d' % (x, cwzm[x]) for x in bin_list(n)]
    return list_of_all_counts

"This is where the user defines what they want to simulate"

"Define the ground state for the desired number qubits"
my_num_qubits = 3
my_ground_state = ggs(my_num_qubits)

"Define a programme consisting of well-defined gates"
#Note that gpg requires square brackets but gcg does not
my_circuit = ["gpg([\"X\",\"I\",\"I\"])",
              "gcg(\"c\",\"I\",\"X\")",
              "gcg(\"c\",\"U(np.pi,1,0,0)\",\"I\")",
              "gpg([\"I\",\"I\",\"H\"])"
              ]

"Define the desired number of shots (i.e. the number of times the circuit is run)"
my_shots = 1000

"This tells Python to print the user's input, run their programme and print the results"
print(my_circuit, end = '\n \n')

print(measure_all(run_programme(my_circuit[::-1], my_ground_state), my_shots))


import pymongo
import json

persistent_client = pymongo.MongoClient("10.4.41.47", 27017)
persistent_zone = persistent_client["PersistentLandingZone"]
formated_zone = persistent_client['FormattedZone']
lookup = persistent_zone['lookup'].find({})
idealista = persistent_zone['idealista'].find({})

def get_lookup(lookup):
    for doc in lookup:
        if doc['filename'] == 'lookup_idealista_extended_2023-05-11-11:31:49.csv':
            lookupIdealista = doc['data']
        else:
            lookupOpenBCN = doc['data']
    return lookupIdealista,lookupOpenBCN

def formatted_idealista(idealista, Formated_zone, lookup ):
    propertyCodes = []
    duplicates = 0
    for document in idealista:
        data = document['data'][0]
        if data['propertyCode'] not in propertyCodes:
            propertyCodes.append(data['propertyCode'])
        else:
            duplicates +=1
            continue

        if 'district' in data.keys():
            district = data['district']
            if 'neighborhood' in data.keys():
                neighborhood = data['neighborhood']
                for neighborhoodLookup in lookup:
                    if neighborhoodLookup['neighborhood'] == neighborhood:

                        idN = neighborhoodLookup['neighborhood_id']
                        idD = neighborhoodLookup['district_id']

                        document['data'][0]['neighborhood_id'] = idN
                        document['data'][0]['district_id'] = idD
                        break
                continue

            for districtLookup in lookup:
                if districtLookup['district'] == district:

                    idD = neighborhoodLookup['district_id']

                    document['data'][0]['district_id'] = idD
                    break

    print(f'Removed {duplicates} duplicates')
lookupIdealista,lookupOpenBCN =  get_lookup(lookup)
formatted_idealista(idealista, formated_zone, lookupIdealista)


import pandas as pd
import plotly.express as px
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score,f1_score,confusion_matrix, plot_confusion_matrix, classification_report
import xgboost as xgb
import shap
shap.initjs()
import matplotlib.pyplot as plt
from pickle import dump
import numpy as np
import dask.dataframe as dd
from dask.distributed import Client
from scipy.stats import ttest_ind
from collections import defaultdict
from tqdm.auto import tqdm
import logging
from umap import UMAP
import hdbscan
from contextlib import nullcontext
from IPython.utils import io
from collections.abc import Iterable
from functools import wraps
from dataclasses import dataclass, field, make_dataclass, asdict
from typing import ClassVar
from functools import partial
from collections import UserDict
import pprint
from typing import Any, Callable
import seaborn as sns
from sklearn.metrics import silhouette_score

#was trying to write custom __eq__ and custom __hash__ to override eq = True, which needs the two instances being compared to be of the exact same class
def dataclass_from_dict(name,input_dict):
#     def __eq__(self, other):
#         inst_fields = self.__dataclass_fields__.keys()
#         return all([self.__dict__[attr] == other.__dict__[attr] for attr in inst_fields])
#     def __hash__(self):
#         inst_fields = self.__dataclass_fields__.keys()
#         return hash(tuple([self.__dict__[attr] for attr in inst_fields]))

    return make_dataclass(name, fields = [(k,type(v)) for k,v in input_dict.items()], frozen=True,) #eq=False, namespace={"__eq__": __eq__, "__hash__": __hash__})

#subclassing dict to extend dict functionalities
class DataClassDict(UserDict):

    def __init__(self, data, cls):
        super(DataClassDict, self).__init__()
        self.data = data
        self._cls = cls
        self._keys =  super(DataClassDict, self).keys

    def __repr__(self):
        from pprint import pformat
        return pformat(str(list(self.keys()))).strip("()")

    def query(self, *args, **kwargs):
        key_instance = self._cls(**kwargs)
        return self.data[key_instance]

    def keys(self):
        return list(self._keys())#[k for k,v in self.data.items()]

    def to_pandas(self, attr: str) -> pd.core.frame.DataFrame:
        def to_pandas_single_df(dc_val, attr):
            col_name = ("|").join([f"func={dc_val.func_name}", *[("=").join([str(k), str(v)]) for k,v in dc_val.args.items()]])
            attr_val = dc_val.return_vals[attr]
            df = pd.DataFrame(attr_val, columns = [col_name] * attr_val.shape[1] if isinstance(attr_val, (np.ndarray, pd.core.frame.DataFrame)) else col_name)
            return df
        return pd.concat([to_pandas_single_df(dc_val, attr) for dc_val in self.values()], axis=1)

#     def to_pandas(self):
#         return pd.DataFrame

def repeated_call(parameters_dict):

    def wrapper(func):
        @wraps(func)
        def _wrapped_func(*args, **kwargs):
            """A wrapper function"""

    return wrapper

def return_with_args(return_vals, filterable=False, return_args=None, pre_process=None,*args, **kwargs):
#     def _preprocess(ele: ):
#         if isinstance(ele, )

    import inspect
    def wrapper(func):
        @wraps(func)
        def _wrapped_func(*args, rename_dict=None, return_vals_filtered=[], **kwargs):
            """A wrapper function"""
            # Extend some capabilities of func
            make_dataclass_mod = partial(make_dataclass, frozen=True, eq=True)
            func_locals_dict = func(*args, **kwargs)
            ##allow user to filter and rename variables
            return_vals_mod = return_vals_filtered if filterable else return_vals
            return_vals_mod = return_vals_mod if rename_dict is None else [rename_dict[val] for val in return_vals_mod]
            return_args_mod = inspect.getargspec(func).args if return_args is None else return_args

            base_class_name = f"{func.__name__}_Result_Base"

            locals()[base_class_name]= make_dataclass_mod(base_class_name, [('args', dict), ('return_vals', dict)])
            created_class = locals()[base_class_name]

            #include the result class within the object

            wrapper_class_name = f"{func.__name__}_Result"
            locals()[wrapper_class_name] = make_dataclass_mod(wrapper_class_name,
                                                                              fields=[('result_class', object), ("fields", list),("func_name", str)], bases=(created_class,))

            created_wrapper_class = locals()[wrapper_class_name]

            args_class_name = f"{func.__name__}_Args"

            return_dict = {
                           "args": {k:v for k,v in func_locals_dict.items() if k in return_args_mod},
                           "return_vals": {k:v for k,v in func_locals_dict.items() if k in return_vals_mod}
                          }

            created_wrapper_class.__repr__ = lambda self: f"<{created_wrapper_class.__name__} object, fields: {list(created_wrapper_class.__dataclass_fields__.keys())}>"
            return_result_instance = created_class(**return_dict)
            return_wrapper_instance = created_wrapper_class(**{"result_class": created_class, "fields" : list(created_class.__dataclass_fields__.keys()), "func_name": func.__name__}, **return_dict)
            return return_wrapper_instance
        return _wrapped_func

    return wrapper

def predict_ensemble(ensemble, X):
    """
    predict_ensemble runs the X data set through
    each classifier in the ensemble list to get predicted
    probabilities.
    Those are then averaged out across all classifiers.
    """
    probs = [r.predict_proba(X)[:,1] for r in ensemble]
    return np.vstack(probs).mean(axis=0)

def adjusted_classes(y_scores, t):
    """
    This function adjusts class predictions based on the prediction threshold (t).
    Will only work for binary classification problems.
    """
    return [1 if y >= t else 0 for y in y_scores]

def print_report(m, X_valid, y_valid, t=0.5, X_train=None, y_train=None, show_output=True):
    """
    print_report prints a comprehensive classification report
    on both validation and training set (if provided).
    The metrics returned are AUC, F1, Precision, Recall and
    Confusion Matrix.
    It accepts both single classifiers and ensembles.
    Results are dependent on the probability threshold t
    applied to individual predictions.
    """
#     X_train = X_train.values
#     X_valid = X_valid.values

    if isinstance(m, list):
        probs_valid = predict_ensemble(m, X_valid)
        y_val_pred = adjusted_classes(probs_valid, t)

        if X_train is not None:
            probs_train = predict_ensemble(m, X_train)
            y_train_pred = adjusted_classes(probs_train, t)
    else:
        probs_valid = m.predict_proba(X_valid)[:,1]
        y_val_pred = adjusted_classes(probs_valid, t)

        if X_train is not None:
            probs_train = m.predict_proba(X_train)[:,1]
            y_train_pred = adjusted_classes(probs_train, t)

    res = [roc_auc_score(y_valid, probs_valid),
           f1_score(y_valid, y_val_pred),
           confusion_matrix(y_valid, y_val_pred)]
    result = f'AUC valid: {res[0]} \nF1 valid: {res[1]}'

    if X_train is not None:
        res += [roc_auc_score(y_train, probs_train),
                f1_score(y_train, y_train_pred)]
        result += f'\nAUC train: {res[3]} \nF1 train: {res[4]}'

    acc_train = m.score(X_train, y_train)
    acc_valid = m.score(X_valid, y_valid)

    if show_output:
        logging.info(f"train acc: {acc_train}")
        logging.info(f"test acc: {acc_valid} ")

        logging.info(result)
        plot_confusion_matrix(m, X_valid, y_valid,
                                     display_labels= y_valid.unique())
        logging.info(classification_report(y_valid, y_val_pred))
        plt.show()
    return {"train":{"AUC": res[3], "F1": res[4], "acc": acc_train}, "test":{"AUC": res[0], "F1": res[1], "acc": acc_valid}}

def train_output_metrics( X,y, *, test_size=0.3, name, path=None, no_seed=False, fitted=None, show_output=True, early_stopping, model=None,**kwargs):
    if no_seed:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,
                                                        stratify=y)
    else:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,random_state=42, stratify=y)

    if not fitted:
        print(f"extra XGBoost kwargs: {kwargs}")
        if not model:
            model = xgb.XGBClassifier(verbosity = 0, **kwargs)#xgb.XGBClassifier(n_estimators=100,  subsample=0.8, colsample_bytree=0.5)
        eval_set = [(X_train, y_train), (X_test, y_test)]
        eval_metric = ["error", "logloss", "auc"]

        # check if multi-label classification
        if "num_class" in kwargs.keys():
            eval_metric += ["merror", "mlogloss"]
        print(X_train.shape, y_train.shape)
        print(eval_metric)

        if early_stopping:
            fitted = model.fit(X_train,y_train, early_stopping_rounds=5, eval_metric=eval_metric,  eval_set=eval_set)

        else:
            fitted = model.fit(X_train,y_train, eval_metric=["error", "logloss", "auc", "aucpr", "map"],  eval_set=eval_set)

        results = model.evals_result()
    if show_output:
        logging.info(f"train size:{1- test_size}, test_size:{test_size}")
        logging.info(f"train case/control num {y_train.value_counts()[1]}/{y_train.value_counts()[0]}")
        logging.info(f"test case/control num  {y_test.value_counts()[1]}/{y_test.value_counts()[0]}")

    metrics_dict = print_report(model, X_test, y_test, t=0.5, X_train=X_train, y_train=y_train, show_output=show_output)

    if path:
        dump(fitted, open(path, 'wb'))

    shap_values_non_zero, important_feat = get_shap_info(model, X, name)

    return {"shap_df": shap_values_non_zero, "important_feat":important_feat, "metrics":metrics_dict, "model":model, "train_index": X_train.index, "test_index": X_test.index}

def get_shap_info(model,X,name):
    explainer = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X)
    shap_values_non_zero = pd.DataFrame(shap_values, columns = X.columns).loc[:, shap_values.sum(axis=0) != 0]
    important_feat = np.abs(shap_values_non_zero).mean().sort_values(ascending=False)

    formatted_name = name.format()
    important_feat.name = f"{name}_shap_vals"
    important_feat = important_feat.to_frame()
    return shap_values_non_zero, important_feat

# def train_multi_params(*args, param_dict:dict=None, func=None, **kwargs):
#     if None in [changing_param, func]:
#         raise ValueError
#     else:
#         results = defaultdict(dict)
#         for key in tqdm(param_dict):
#             for val in tqdm(param_dict[key]):
#                 new_dict = kwargs.copy()
#                 new_dict[key] = val
#                 results[key][val] = func(*args,**new_dict)

def repeat_train(X, y, num_times, different_train_test_split, name, model=None, silent=False, **kwargs):
    result_list =[]
    print(f"Extra params: {kwargs}")
    for i in tqdm(range(num_times)):
        with io.capture_output() if silent else nullcontext() as captured :
            result_dict = train_output_metrics(X, y, name="base_model",no_seed=different_train_test_split, early_stopping=True, model=model, **kwargs)
            result_list.append(result_dict)
    auc_list = [metric_dict["metrics"]["test"]["AUC"] for metric_dict in result_list]
    auc_df = pd.DataFrame({"AUC": auc_list, "dataset":name})
    fig = px.violin(auc_df, box=True, y = "AUC", color = "dataset")
    return {"AUC":auc_df, "result_list": result_list, "fig": fig}

def plot_curve(model_evals_result):
    results = model_evals_result
    epochs = len(results['validation_0']['error'])
    x_axis = range(0, epochs)
    # plot log loss
    train_df = pd.DataFrame.from_dict(base_model.evals_result()["validation_0"], orient="columns")
    train_df["set"] = "train"
    test_df = pd.DataFrame.from_dict(base_model.evals_result()["validation_1"], orient="columns")
    test_df["set"] = "test"
    combined = pd.concat([train_df, test_df])

    for column in combined.drop(columns="set").columns:
        fig = px.line(combined, x = combined.index, y=column, color ="set")
        fig.show()
#         fig, ax = plt.subplots()
#         ax.plot(x_axis, results['validation_0'][metric], label='Train')
#         ax.plot(x_axis, results['validation_1'][metric], label='Test')
#         ax.legend()
#         plt.ylabel(f'{metric}')
#         plt.title(f'XGBoost {metric}')
#         plt.show()

@return_with_args(return_vals=["multi_param_result_dict"])
def draw_umap(data,n_neighbors=None, min_dist=None, c=None,  n_components=None, metric='euclidean', title="Untitled_exp", plot=True, plotly_3D = True, return_vals_filtered=["u","mapper"], umap_kwargs={}, **kwargs):
    @return_with_args(return_vals=["u", "mapper"], filterable = True)
    def draw_one_umap(one_n_neighbor, one_min_dist, one_n_component):
        fit = UMAP(
        n_neighbors= one_n_neighbor,
        min_dist= one_min_dist,
        n_components=one_n_component,
        metric=metric,
        random_state = 42,
        **umap_kwargs
    )

        mapper = fit.fit(data);
        u = fit.transform(data);

        fig = None
        final_title = f"{title}: {one_n_component}D representation, n_neighbor: {one_n_neighbor}, min_dist: {one_min_dist}"

        if plot:
#             if one_n_component == 1:
#                 ax = fig.add_subplot(111)
#                 ax.scatter(u[:,0], range(len(u)), c=c)
            fig = plt.figure()

            if one_n_component == 2:
                ax = fig.add_subplot(111)
                scatter = ax.scatter(u[:,0], u[:,1], c=c, label=c)
                plt.title(final_title, fontsize=18)
                legend = ax.legend(*scatter.legend_elements())
                ax.add_artist(legend)
                plt.show()
            if one_n_component == 3:

#                 ax = fig.add_subplot(111, projection='3d')
#                 scatter = ax.scatter(u[:,0], u[:,1], u[:,2], c=c, s=100)

                if plotly_3D:
                    plotly_fig_3D = px.scatter_3d(x=u[:,0], y=u[:,1], z=u[:,2],color=c, opacity = 0.3, title = final_title, **kwargs)
                    plotly_fig_3D.update_layout(legend=dict(
                                    orientation="h",
                                    yanchor="bottom",
                                    y=1.02,
                                    xanchor="right",
                                    x=1
                                ))

                    display(plotly_fig_3D)

        u = pd.DataFrame(u, columns = [f"UMAP_{one_n_component}D_{i}" for i in range(1, one_n_component+1)], index=data.index)
        return locals()

    standardize_input = lambda x, default: list(x) if isinstance(x, Iterable) else default if x is None else [x]

    n_neighbor_list = standardize_input(n_neighbors, [5,10, 20, 50, 100, 200])
    min_dist_list = standardize_input(min_dist, [0, 0.1])
    n_component_list = standardize_input(n_components, [2,3])

    multi_param_result_dict = {}

    MultiParamClass = None

    for one_min_dist in min_dist_list:
        for one_n_neighbor in n_neighbor_list:
            if one_n_neighbor >= data.shape[0]:
                continue
            for one_n_component in n_component_list:
                returned_instance = draw_one_umap(one_n_neighbor, one_min_dist, one_n_component, return_vals_filtered=return_vals_filtered)

                if MultiParamClass is None:
                    MultiParamClass = dataclass_from_dict("Params", returned_instance.args)

                dict_key = MultiParamClass(**returned_instance.args)
                multi_param_result_dict[dict_key] = returned_instance

                #multi_param_result_dict[frozenset(returned_instance.args.items())] = returned_instance
    multi_param_result_dict = DataClassDict(multi_param_result_dict, MultiParamClass)
    return  locals()#multi_param_result_dict #locals() #{"embedding": u, "mapper": mapper}

def cluster_hdbscan(clusterable_embedding, min_cluster_size, viz_embedding_list, plot=True):
    print(f"min_cluster size: {min_cluster_size}")
    clusterer= hdbscan.HDBSCAN(
        min_cluster_size=min_cluster_size,
    ).fit(clusterable_embedding)
    labels = clusterer.fit_predict(clusterable_embedding)
    print(f"found {len(np.unique(labels))} clusters")
    clustered = (labels >= 0)
    frac_clustered = np.sum(clustered)/labels.shape[0]
    print(f"fraction clustered: {np.sum(clustered)/labels.shape[0]}")
    clusterer.condensed_tree_.plot(select_clusters=True,
                               selection_palette=sns.color_palette('deep', 8))

    for embedding in viz_embedding_list:

        if embedding.shape[1] == 3:
            fig = px.scatter_3d(x= embedding[:, 0],
                y= embedding[:, 1],
                z= embedding[:, 2],
                color = labels.astype("str"))
            display(fig)
            #.show()
#             ax = fig.add_subplot(111, projection='3d')

#             ax.scatter(embedding[~clustered][:, 0],
#                 embedding[~clustered][:, 1],
#                 embedding[~clustered][:, 2],
#                 c=(0.5, 0.5, 0.5),
#                 alpha=0.5)
#             ax.scatter(embedding[clustered][:, 0],
#                     embedding[clustered][:, 1],
#                     embedding[~clustered][:, 2],
#                     c=labels[clustered],
#                     cmap='Spectral')
        else:
            fig = px.scatter(x= embedding[:, 0],
                y= embedding[:, 1],
                color = labels.astype("str"))
            display(fig)
#             fig = plt.figure()
#             ax = fig.add_subplot(111)
#             ax.scatter(embedding[~clustered][:, 0],
#                 embedding[~clustered][:, 1],
#                 c=(0.5, 0.5, 0.5),
#                 s=10,
#                 alpha=0.5)
#             ax.scatter(embedding[clustered][:, 0],
#                     embedding[clustered][:, 1],
#                     c=labels[clustered].reshape(-1,1),
#                     s=10,
#                     cmap='Spectral')

#             plt.title(f"min_cluster_size = {min_cluster_size}", fontsize=18)
#             legend = ax.legend(*scatter.legend_elements())
#             ax.add_artist(legend)
#             plt.show()

    return {"labels":labels, "clusterer": clusterer}

def get_all_cluster_settings(fitted_hdbscan, embedding_used_to_cluster, unclustered_threshold=0.2, extra_cols_dict={}):
    '''
    unclustered_threshold: fraction of unclustered points allowed, if the number of points is more than this threshold, the setting is discarded
    '''
    test_label_dict = {}
    silhouette_scores = set()
    df_list = []
    linkage_tree = fitted_hdbscan.single_linkage_tree_
    condensed_tree_df = fitted_hdbscan.condensed_tree_.to_pandas().query("child_size > 1")
    for i, lambda_val in tqdm(enumerate(linkage_tree.to_pandas().sort_values("size", ascending = False).query("size > @condensed_tree_df.child_size.min()").distance.unique() - 0.01)):
        for child_size in sorted(condensed_tree_df["child_size"].unique() - 1):
            test_labels = fitted_hdbscan.single_linkage_tree_.get_clusters(lambda_val, min_cluster_size=2)
            # remove settings where alot is unclustered
            if (test_labels == -1).sum()/len(test_labels) > unclustered_threshold:
                continue
            try:
                #print(str(len(np.unique(test_labels))))
                test_silhouette_score = silhouette_score(embedding_used_to_cluster, test_labels)
            except ValueError:
                #print("skipped")
                continue

            if test_silhouette_score in silhouette_scores:
                continue

            col_name = str(len(np.unique(test_labels)))

            j = 0
            if col_name in test_label_dict:
                while f"{col_name}_{j}" in test_label_dict:
                    j+=1
                col_name = f"{col_name}_{j}"

            silhouette_scores.add(test_silhouette_score)
            #test_labels_df = pd.DataFrame(test_labels, columns=[f"{len(np.unique(test_labels))}_clusters_label"]).assign(**)
            df_list.append({"cluster_setting_num": i,
                            "col_name" : f"{col_name}_clusters_label",
                            "silhouette_score": test_silhouette_score,
                            "num_clusters": len(np.unique(test_labels)),
                            })#test_labels_df[[f"{len(np.unique(test_labels))}_clusters_label"]])
            test_label_dict[col_name] = test_labels
            #display(test_labels_df.head())
            #test_label_dict.append({f"{len(np.unique(test_labels))}_cluster_labels":list(test_labels), "silhouette_score": silhouette_score(test_clusteval_embed, test_labels) })

    all_labels_df = pd.DataFrame.from_dict(test_label_dict)

    #sort by number of clusters
    all_labels_df = all_labels_df[sorted(all_labels_df.columns, key=lambda x: tuple(map(int,x.split('_'))))].add_suffix("_clusters_label")
    all_labels_df = all_labels_df.assign(**extra_cols_dict)
    all_labels_df.index = embedding_used_to_cluster.index
    display(all_labels_df.head())
    cluster_metrics_df = pd.DataFrame(df_list)#.sort_values(["silhouette_score","num_clusters"], ascending= [False, True])

    return {"all_labels_df": all_labels_df, "cluster_metrics_df": cluster_metrics_df}

def get_cluster_defining_features(X, clustering_label, cluster_setting_name):
    """Need to provide X as features only and NO clustering label """
    cluster_names = np.unique(clustering_label)
    final_dict = {}
    all_cluster_info_mean_peaks_per_cluster = []

    for cluster_name in tqdm(cluster_names):
        if cluster_name == -1:
            continue
        print(cluster_name)
        final_dict[cluster_name] = {}
        cluster_one_v_all_labels = (clustering_label == cluster_name).astype(int)
        #use scale_pos_weight to address imbalance between clustering class vs rest
        with io.capture_output() as captured:
            all_OnevsAll_models_result_dict = repeat_train(X, cluster_one_v_all_labels,num_times=1000, different_train_test_split=True, name=cluster_setting_name, scale_pos_weight=cluster_one_v_all_labels.value_counts()[0]/cluster_one_v_all_labels.value_counts()[1], show_output=False)

        final_dict[cluster_name]["AUC_fig"] = all_OnevsAll_models_result_dict["fig"]

        #breakpoint()
        shap_mean_peaks_per_cluster_all_models = (pd.concat([result["shap_df"].assign(model_num=i) for i, result in enumerate(all_OnevsAll_models_result_dict["result_list"]) if result["metrics"]["test"]["AUC"] > 0.9]).reset_index())
        #explainer = shap.TreeExplainer(shap_mean_peaks_per_cluster_list[0][2])

        final_dict[cluster_name]["shap_summary_plot"]= shap.summary_plot(shap_mean_peaks_per_cluster_all_models.drop(columns=["model_num"]).groupby("index").mean().values, X[shap_mean_peaks_per_cluster_all_models.drop(columns=["index", "model_num"]).columns])

        aggregate_peak_info_mean_peaks_per_cluster = X[shap_mean_peaks_per_cluster_all_models.drop(columns=["index", "model_num"]).columns].assign(is_cluster = np.where(cluster_one_v_all_labels ==1, "cases_in_cluster", "cases_not_in_cluster")).groupby("is_cluster").agg(["mean", "median"]).swaplevel(-2,-1, axis=1)

        aggregate_peak_info_mean_peaks_per_cluster = pd.concat([aggregate_peak_info_mean_peaks_per_cluster["mean"].T.add_suffix("_mean"), aggregate_peak_info_mean_peaks_per_cluster["median"].T.add_suffix("_median")], axis=1)
        aggregate_peak_info_mean_peaks_per_cluster

        cluster_info_mean_peaks_per_cluster = shap_mean_peaks_per_cluster_all_models.drop_duplicates("model_num").drop(columns=["index", "model_num"]).count().to_frame()
        cluster_info_mean_peaks_per_cluster.columns = ["num_model_used_this_peak"]
        cluster_info_mean_peaks_per_cluster = cluster_info_mean_peaks_per_cluster.merge(aggregate_peak_info_mean_peaks_per_cluster, how = "outer", left_index = True, right_index=True)
        cluster_info_mean_peaks_per_cluster["cluster"] = cluster_name

        final_dict[cluster_name]["cluster_info_df"] = cluster_info_mean_peaks_per_cluster
        all_cluster_info_mean_peaks_per_cluster.append(cluster_info_mean_peaks_per_cluster)

    final_dict["combined_cluster_info_df"] = pd.concat(all_cluster_info_mean_peaks_per_cluster)
    final_dict["cluster_setting"] = cluster_setting_name
    return final_dict