from math import cos
from PIL import Image, ImageDraw
from math import radians,cos,sin

def calcule_courbe_de_Weierstrass(x):
    somme=cos(x)
    for i in range(1,500):
        somme= somme+ (1/(2**i))*cos(x*(4**i))
    print(somme)

def wie(x):
    return cos(x)+(1)/(2) *cos(4*x)+(1)/(4)* cos(16*x)+(1)/(8)* cos(64* x)+(1)/(16) *cos(256* x)+\
            (1)/(32) *cos(32*32* x)+(1)/(64) *cos(64*64 *x)+(1)/(128) *cos(128*128 *x)+(1)/(256) *cos(256*256 *x)

class l_syst:
    def __init__(self,Variable,Constantes,Axiome,Règle,angle=90,avance=20,
    img_taille=(10000,10000),font=(0,0,0),coul=(255,255,255),
    depart=(5000,5000),épaisseur_dessin=1,nom="l_système",dirrection=0):
        """
        variable :
            type liste
            variable dans le l-système
        constante :
            type liste
            constante
        axiome:
            type sting
            axiome de départ
        règle :
            type liste
            (x,y,u,v,.....)
            x -> y , u -> v ,.....
        angle:
            type entier
            angle pour certaine constante
        avance:
            type entier
            longeur de deplacement
        """
        self.angle=-angle
        self.actuel=Axiome
        self.règle=Règle
        self.constante=Constantes
        self.variable=Variable
        self.avance=avance
        self.pas_actuel=0
        self.__creation_variable__()
        self.direction=0
        self.coul=coul
        self.image_taille=img_taille
        self.font=font
        self.depart=depart
        self.épaisseur=épaisseur_dessin
        self.nom=nom
        self.__verification_constante__()
        self.direction=-dirrection
    def __verification_constante__(self):
        liste_constante=["+","-","&","^","<",">","|","[","]"]
        for i in self.constante:
            if i not in liste_constante:
                raise ValueError("constante non existante")

    def __creation_variable__(self):
        variable_remplacement=["╚","╔","╩","╦","╠","═","╬","é","è","ë","ê","æ",
                               "û","ü","µ","à","â","ä","ç","ï","î","ô","ö","_",
                               "ÿ","$","£","¤","Ê","Â","Î","Ô","Û","Ë","Ä","Ü",
                               "Ï","Ö","©","®","Æ","¿","░","▒","▓","ð","Ð","¶",
                               "ı","┘","┌","█","▄","¦","Ì","▀","±","‗","ß","þ",
                               "Þ","½","¼","¾","│","┤"]
        indice_remplacement=0
        remplacement_definitif=[]
        for i in self.variable:
            if i in variable_remplacement:
                del variable_remplacement[variable_remplacement.index(i)]
        for i in self.variable:
            for j in range(0,len(self.règle),2):
                if i in self.règle[j]:
                    remplacement_definitif.append((i,variable_remplacement[indice_remplacement]))
                    self.règle[j]=variable_remplacement[indice_remplacement]
                    indice_remplacement+=1
        self.remplacement=remplacement_definitif

    def __maj_sense__(self,angle):
        self.direction=(self.direction+angle)%360

    def avance_de_1 (self):
        actuel=self.actuel
        for i in range (len(self.remplacement)):
                actuel=actuel.replace(self.remplacement[i][0],self.remplacement[i][1])
        for i in range (0,len(self.règle),2):
            actuel=actuel.replace(self.règle[i],self.règle[i+1])
        self.actuel=actuel
        self.pas_actuel+=1

    def translation(self,a,angle,longeur):
        angle_radian=radians(angle)
        return (a[0]+longeur*cos(angle_radian),a[1]+longeur*sin(angle_radian))

    def tracer(self):
        img=Image.new("RGB",self.image_taille,self.font)
        dessin=ImageDraw.Draw(img)
        a_tracer=self.actuel
        for i in self.variable:
            if i !="F":
                a_tracer=a_tracer.replace(i,"")
        point_actuel=self.depart
        for i in a_tracer:
            if i =="F":
                nouveaux_point=self.translation(point_actuel,self.direction,self.avance)
                dessin.line((point_actuel,nouveaux_point),self.coul,self.épaisseur)
                point_actuel=nouveaux_point
            if i == "+":
                self.__maj_sense__(self.angle)
            if i =="-":
                self.__maj_sense__(-self.angle)
            if i == "[":
                point_sauv=point_actuel
            if i == "]":
                point_actuel=point_sauv
            if i == "|":
                self.__maj_sense__(180)
            if i == "&":
                if self.direction<=90 and self.direction>=-90:
                    self.__maj_sense__(-self.angle)
                else:
                    self.__maj_sense__(self.angle)
            if i == "^":
                if self.direction<=90 and self.direction>=-90:
                    self.__maj_sense__(self.angle)
                else:
                    self.__maj_sense__(-self.angle)
            if i=="<":
                angle_sauv=self.direction
                self.__maj_sense__(self.angle)
                nouveaux_point=self.translation(point_actuel,self.direction,self.avance)
                dessin.line((point_actuel,nouveaux_point),self.coul,self.épaisseur)
                point_actuel=nouveaux_point
                self.direction=angle_sauv
            if i==">":
                angle_sauv=self.direction
                self.__maj_sense__(-self.angle)
                nouveaux_point=self.translation(point_actuel,self.direction,self.avance)
                dessin.line((point_actuel,nouveaux_point),self.coul,self.épaisseur)
                point_actuel=nouveaux_point
                self.direction=angle_sauv
        img.save("{}.jpg".format(self.nom))

"""
    F : Se déplacer d’un pas unitaire (Variable);
    + : Tourner à gauche d’angle α (Constante);
    - : Tourner à droite d’un angle α  (Constante);
    & : Pivoter vers le bas d’un angle α(Constante);
    ^ : Pivoter vers le haut d’un angle α  (Constante);
    < : Roulez vers la gauche d’un angle α  (Constante);
    > : Roulez vers la droite d’un angle α (Constante);
    | : Tourner sur soi-même de 180°  (Constante);
    [ : Sauvegarder la position courante (Constante);
    ] : Restaurer la dernière position sauvée (Constante).
"""
##a=l_syst(["F","Y","X"],["+","-"],"FX",["X","X+YF+","Y","-FX-Y"],avance=20,épaisseur_dessin=2)
##a=l_syst(["F"],["+","-","[","]"],"F",["F","+F−−F+"],45,épaisseur_dessin=2,depart=(7500,2500))
##a=l_syst(["F"],["+","-"],"F",["F","F+F-F"],120,avance=5,épaisseur_dessin=2,img_taille=(20000,20000))
##a=l_syst(["F"],["+","-"],"F",["F","F+F--F+F"],60,avance=10,épaisseur_dessin=2,img_taille=(30000,10000),depart=(2500,9900))
##a=l_syst(["F","f"],["+","-"],"F+F+F+F",["F","F+f-FF+F+FF+Ff+FF-f+FF-F-FF-Ff-FFF","f","ffffff"],avance=20,épaisseur_dessin=2)

for i in range(10):
    a.avance_de_1()
a.tracer()

l_système/l-systeme avec remplacement v2.py


from collections import deque
import sys

input = lambda : sys.stdin.readline().rstrip()

N, M = map(int, input().split())
graph = [[] for _ in range(N+1)]

# 각 과목의 선수과목 개수
indegree = [0] * (N+1)

# 각 과목을 수강하는데 걸리는 시간
time = [1] * (N+1)

for _ in range(M):
    A, B = map(int, input().split())
    # A-> B
    graph[A].append(B)
    # B의 선수과목 개수 추가
    indegree[B] += 1

q = deque()
# 선수과목이 없는 과목들을 큐에 넣음
for x in range(1, N+1):
    if indegree[x] == 0:
        q.append(x)

while q:
    now = q.popleft()
    for x in graph[now]:
        # 과목을 수강하는데 걸리는 시간 갱신
        # time[x] : 기존 수강 걸리는 시간
        # time[now] + 1 : now 과목이 수강하는데 걸리는 시간 + 한 학기 이후
        time[x] = max(time[x], time[now]+1)
        indegree[x] -= 1
        # x가 더이상 들어야할 선수과목이 없으면 큐에 넣음
        if indegree[x] == 0:
            q.append(x)

for x in time[1:]:
    print(x, end=" ")


#!/usr/bin/env python
# -*- coding:utf-8 -*-

from flask import Flask,jsonify,request,abort,Response
from time import strftime,localtime
from uuid import uuid4
import json

# 实例化http服务
app = Flask(__name__)

# 业务逻辑
class Task(object):
    def __init__(self,content):
        self.id = str(uuid4())
        self.content = content
        self.created_at = strftime("%y-%m-%d:%H:%M:%S")
        self.is_finished = False
        self.finished_at = None

    def finish(self):
        self.is_finished = True
        self.finished_at = strftime("%y-%m-%d:%H:%M:%S")

    def json(self):
        return {
            'id':self.id,
            'content':self.content,
            'created_at':self.created_at,
            'is_finished':self.is_finished,
            'finished_at':self.finished_at,
        }

# 模拟数据库接口
tasks_db = {'0':Task('test')}
get_task = tasks_db.get

# 接收请求和做出响应
# 展示
@app.route('/task')
def show():
    return jsonify(data=[task.json() for task in tasks_db.values()])

# 添加
@app.route('/task',methods=['POST'])
def add():
    content = request.form.get('content',None)
    if not content:
        abort(400)
    task = Task(content)
    tasks_db[task.id] = task
    return Response()

# 修改
@app.route('/task/<tid>/finish',methods=['PUT'])
def finish(tid):
    task = get_task(tid)
    if task:
        task.finish()
        tasks_db[tid] = task
        return Response()
    abort(404)

# 删除
@app.route('/task/<tid>',methods=['DELETE'])
def delete(tid):
    task = get_task(tid)
    if task:
        tasks_db.pop(tid)
        return Response()
    abort(404)

if __name__ == '__main__':
    app.run(host='0.0.0.0',port=8000)


from sudoku import Sudoku

def main():
    game = Sudoku()  # Creating instance of Sudoku
    game.setDifficulty(1)  # Setting difficulty
    game.buildpuzzle()  # Building the puzzle
    game.play()  # Playing the puzzle by setting and getting cell values

if __name__ == '__main__':
    main()


import h5py
import os
import torch
import numpy as np
import torch.optim as optim
from torch.utils import data
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import pandas as pd

import datetime
import matplotlib.animation as animation
import matplotlib.pyplot as plt
import re
from io import StringIO
import ffmpeg
import pdb
from sklearn.metrics import (
    classification_report,
    roc_auc_score,
    confusion_matrix,
    roc_curve,
    f1_score,
    auc,
    precision_recall_curve,
)
import pdb

"""
 Function that takes inputs (sample, reconstruction, and label)
 generates reconstruction erros, and then generates performance metrics and saves these in a csv

performance metrics
- AUC ROC and PR for std, and mean of frame error (both day and night)
- AUC ROC and PR for std and mean of window error for different thresholds (both day and night)

"""

def get_performance_metrics(sample, output, labels, window_len):
    window_std, window_mean, window_labels = get_window_metrics(sample, output, labels, window_len)
    frame_std, frame_mean, frame_labels = get_frame_metrics(sample, output, labels, window_len)
    return(frame_std, frame_mean, frame_labels, window_std, window_mean, window_labels)

def get_window_metrics(sample, output, labels, window_len):
    recon_data = output.reshape(output.shape[1], window_len, 64 * 64)
    sample_data = sample.reshape(sample.shape[1], window_len, 64 * 64)
    labels = shape_labels(labels)

    recon_error = np.mean(np.power(sample_data - recon_data, 2), axis=2)
    mean_window_error = []
    std_window_error = []
    window_labels = []
    for tolerance in range(1, window_len+1):
        stride = 1
        windowed_labels = create_windowed_labels(labels, stride, tolerance, window_len)
        windowed_labels = windowed_labels[:,0]
        inwin_mean = np.mean(recon_error, axis =1)
        inwin_std = np.std(recon_error, axis =1)
        mean_window_error.append(inwin_mean)
        std_window_error.append(inwin_std)
        window_labels.append(windowed_labels)
    return(mean_window_error, std_window_error, window_labels)

def get_frame_metrics(sample, output, labels, window_len):
    recon_data = output.reshape(output.shape[1], window_len, 64 * 64)
    sample_data = sample.reshape(sample.shape[1], window_len, 64 * 64)
    labels = shape_labels(labels)

    recon_error = np.mean(np.power(sample_data - recon_data, 2), axis=2)

    # ------- Frame Reconstruction Error ---------------
    # create empty matrix w/ orignal number of frames
    mat = np.zeros((len(recon_error) + window_len - 1, len(recon_error)))
    mat[:] = np.NAN
    # dynmaically fill matrix with windows values for each frame
    #print(len(recon_error))
    for i in range(len(recon_error)):
        win = recon_error[i]
        #print(len(win))
        #print(i)
        mat[i : len(win) + i, i] = win
    frame_scores = []
    # each row corresponds to a frame across windows
    # so calculate stats for a single frame frame(row)
    for i in range(len(mat)):
        row = mat[i, :]
        mean = np.nanmean(row, axis=0)
        std = np.nanstd(row, axis=0)
        frame_scores.append((mean, std, mean + std * 10 ** 3))

    frame_scores = np.array(frame_scores)
    x_std = frame_scores[:, 1]
    x_mean = frame_scores[:, 0]

    return(x_mean, x_std, labels)

def get_total_performance_metrics(frame_stats, window_stats, window_len):
    get_curves_and_thresholds(frame_stats, window_stats, window_len)

    video_metrics = np.zeros((len(frame_stats), 5, window_len ))
    print(video_metrics.shape)

    # here i need to get the error and everything and store it in video metrics

    for i in range(len(frame_stats)):
        # print(i)
        # this a single video metrics
        frame_mean, frame_std, frame_labels = frame_stats[i]
        mean_window_error, std_window_error, window_labels = window_stats[i]

        # store frame results for this vidoe
        video_metrics[i, 0, 0], video_metrics[i, 1, 0], video_metrics[i, 2, 0], video_metrics[i, 3, 0] = get_performance_values(frame_mean, frame_std, frame_labels)
        video_metrics[i, 4, 0] = 0

        # store each thresholds results for this video
        for j in range(1, window_len):
            vid_labels = window_labels[j-1]
            #print(vid_labels)
            if len(np.unique(vid_labels)) != 2:
                # print("ERROR: no fall for threshold {} in video {}".format(j, i))
                continue
            video_metrics[i, 0, j], video_metrics[i, 1, j], video_metrics[i, 2, j], video_metrics[i, 3, j] = get_performance_values(mean_window_error[j-1], std_window_error[j-1], vid_labels)
            video_metrics[i, 4, j] = j

    video_metrics[video_metrics == 0] = np.nan
    final_performance_mean = np.nanmean(video_metrics, axis=0) # get the mean performance across all videos
    final_performance_std = np.nanstd(video_metrics, axis=0)

    #np.savetxt('results.csv', final_performance, delimiter=',', fmt='%d')

    pd.DataFrame(final_performance_mean).to_csv("results_mean.csv")
    pd.DataFrame(final_performance_std).to_csv("results_std.csv")

    return(final_performance_mean, final_performance_std)

def get_curves_and_thresholds(frame_stats, window_stats, window_len):
    frame_mean_flat = []
    frame_std_flat = []
    frame_labels_flat = []
    for i in range(len(frame_stats)):
        # print(i)
        # this a single video metrics
        frame_mean, frame_std, frame_labels = frame_stats[i]
        frame_mean_flat.extend(frame_mean)
        frame_std_flat.extend(frame_std)
        frame_labels_flat.extend(frame_labels[:len(frame_std)])

    def plot_ROC_AUC(fpr, tpr, roc_auc, data_option):
        plt.figure()
        lw = 2
        plt.plot(fpr, tpr, color='darkorange',
            lw=lw, label='ROC curve (area = %0.4f)' % roc_auc)
        plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title('Receiver Operating Characteristic for {}'.format(data_option))
        plt.legend(loc="lower right")
        plt.savefig('graphs/{}ROC_AUC.png'.format(data_option))

    def plot_PR_AUC(precision, recall, mean_AUPR, no_skill, data_option):
        plt.figure()
        plt.plot([0,1], [no_skill,no_skill], linestyle='--', label='No Skill (area = %0.4f)' % no_skill)
        plt.plot(recall, precision, marker='.', label='AUPR curve (area = %0.4f)' % mean_AUPR)
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, max(precision)/5])
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.title('Precision Recall Curve for {}'.format(data_option))
        plt.legend(loc="lower right")
        plt.savefig('graphs/{}PR_AUC.png'.format(data_option))

    # calculate metrics for Standard Deviation
    roc_fpr, roc_tpr, roc_thresholds = roc_curve(y_true=frame_labels_flat, y_score=frame_std_flat, pos_label=1)
    std_AUROC = auc(roc_fpr, roc_tpr)
    data_option = 'STD of Reconstruction Error'
    plot_ROC_AUC(roc_fpr, roc_tpr, std_AUROC, data_option)

    precision, recall, pr_thresholds = precision_recall_curve(frame_labels_flat, frame_std_flat, pos_label=1)
    data_option = 'STD of Reconstruction Error'
    no_skill = frame_labels_flat.count(1) / len(frame_labels_flat)
    mean_AUPR = auc(recall, precision )
    plot_PR_AUC(precision, recall, mean_AUPR, no_skill, data_option)

    '''
    gmeans = np.sqrt(roc_tpr * (1-roc_fpr))
    ix = np.argmax(gmeans)
    print('Best Threshold=%f, G-Mean=%.3f' % (roc_thresholds[ix], gmeans[ix]))
    optimal_threshold = roc_thresholds[ix]
    '''

    optimal_idx = np.argmax(roc_tpr - roc_fpr)
    optimal_threshold = roc_thresholds[optimal_idx]

    for i in range(len(frame_std_flat)):
        if frame_std_flat[i] >= optimal_threshold:
            frame_std_flat[i] = 1
        else:
            frame_std_flat[i] = 0

    tn, fp, fn, tp = confusion_matrix(frame_labels_flat, frame_std_flat).ravel()
    TPR = tp/(tp+fn)
    FPR = fp/(fp+tn)
    Precision = tp/(tp+fp)
    Recall = tp/(tp+fn)
    print("----------------------------------")
    print("STD Global Classification Results")
    print('TPR {:.3f}, FPR {:.3f}, Precision {:.3f}, Recall {:.3f}'.format(TPR, FPR, Precision, Recall))
    print('tn {}, fp {}, fn {}, tp {}'.format(tn, fp, fn, tp))
    print("std_AUROC  {:.3f}".format(std_AUROC))
    print("----------------------------------")

    # calculate the Mean AUC
    roc_fpr, roc_tpr, roc_thresholds = roc_curve(y_true=frame_labels_flat, y_score=frame_mean_flat, pos_label=1)
    mean_AUROC = auc(roc_fpr, roc_tpr)
    data_option = 'Mean of Reconstruction Error'
    plot_ROC_AUC(roc_fpr,roc_tpr, mean_AUROC, data_option)

    precision, recall, thresholds = precision_recall_curve(frame_labels_flat, frame_mean_flat, pos_label=1)
    data_option = 'Mean of Reconstruction Error'
    no_skill = frame_labels_flat.count(1) / len(frame_labels_flat)
    mean_AUPR = auc(recall, precision )
    plot_PR_AUC(precision, recall, mean_AUPR, no_skill, data_option)

    optimal_idx = np.argmax(roc_tpr - roc_fpr)
    optimal_threshold = roc_thresholds[optimal_idx]

    for i in range(len(frame_mean_flat)):
        if frame_mean_flat[i] >= optimal_threshold:
            frame_mean_flat[i] = 1
        else:
            frame_mean_flat[i] = 0

    tn, fp, fn, tp = confusion_matrix(frame_labels_flat, frame_mean_flat).ravel()
    TPR = tp/(tp+fn)
    FPR = fp/(fp+tn)
    Precision = tp/(tp+fp)
    Recall = tp/(tp+fn)
    print("----------------------------------")
    print("Mean Global Classification Results")
    print('TPR {:.3f}, FPR {:.3f}, Precision {:.3f}, Recall {:.3f}'.format(TPR, FPR, Precision, Recall))
    print('tn {}, fp {}, fn {}, tp {}'.format(tn, fp, fn, tp))
    print("mean_AUROC {:.3f}".format(mean_AUROC))
    print("----------------------------------")

    return()

def get_performance_values(vid_mean, vid_std, vid_labels):

    # calculate metrics for Standard Deviation
    std_fpr, std_tpr, std_thresholds = roc_curve(y_true=vid_labels[:len(vid_std)], y_score=vid_std, pos_label=1)
    std_AUROC = auc(std_fpr, std_tpr)

    std_precision, std_recall, thresholds = precision_recall_curve(vid_labels[:len(vid_std)], vid_std)
    std_AUPR = auc(std_recall, std_precision)

    '''
    optimal_idx = np.argmax(std_tpr - std_fpr)
    optimal_threshold = std_thresholds[optimal_idx]
    for i in range(len(vid_std)):
        if vid_std[i] >= optimal_threshold:
            vid_std[i] = 1
        else:
            vid_std[i] = 0

    print("Optimal Mean Threshold value is:", optimal_threshold)
    std_f1 = f1_score(vid_labels[:len(vid_std)], vid_std)
    print('individual F1', std_f1)
    tn, fp, fn, tp = confusion_matrix(vid_labels[:len(vid_std)], vid_std).ravel()
    print(tn, fp, fn, tp)
    '''

    # calculate the Mean AUC
    fpr, tpr, thresholds = roc_curve(y_true=vid_labels[:len(vid_std)], y_score = vid_mean, pos_label=1)
    mean_AUROC = auc(fpr, tpr)

    mean_precision, mean_recall, thresholds = precision_recall_curve(vid_labels[:len(vid_std)], vid_mean)
    mean_AUPR = auc(mean_recall, mean_precision)
    data_option = 'Mean Err'

    #print('-------------------------------------')
    return std_AUROC, mean_AUROC, std_AUPR, mean_AUPR

def shape_labels(labels):
    # generate labels
    label = labels[0, :, :]
    windowed_labels = label # shape (window_len, # of windows)
    frame_labels = un_window(label)
    return frame_labels

def create_windowed_labels(labels, stride, tolerance, window_length):
    output_length = int(np.floor((len(labels) - window_length) / stride))+1
    output_shape = (output_length, 1)
    total = np.zeros(output_shape)
    i=0
    while i < output_length:
        next_chunk = np.array([labels[i+j] for j in range(window_length)])
        num_falls = sum(next_chunk) #number of falls in the window

        if num_falls >= tolerance:
            total[i] = 1
        else:
            total[i] = 0

        i = i+stride
    labels_windowed = total
    return labels_windowed

def un_window(windowed_data):
# Input: Windowed Data with format (window_length, # of windows )
    unwindowed_data = np.zeros(windowed_data.shape[0] + windowed_data.shape[1])
    for i in range(len(unwindowed_data)):
        if i >= windowed_data.shape[1]:
            last_window = windowed_data[:, i - 1]
            unwindowed_data[i:] = last_window
            break
        else:
            unwindowed_data[i] = windowed_data[0, i]

    return unwindowed_data

def animate(test_data, recons_seq, frame_mean, dset, start_time):
    ani_dir = "./Animation/{}/".format(dset)
    ani_dir = ani_dir + "/{}".format(start_time)
    if not os.path.isdir(ani_dir):
        os.makedirs(ani_dir)
    print("saving animation to {}".format(ani_dir))

    animate_fall_detect_present(
        testfall=test_data[:, 0, :].reshape(len(test_data), 64, 64, 1),
        recons=recons_seq[:, 0, :].reshape(len(recons_seq), 64, 64, 1),
        win_len=1,
        scores=frame_mean,
        to_save=ani_dir + "/{}.mp4".format(len(test_data)),
    )

def animate_fall_detect_present(
    testfall, recons, scores, win_len, threshold=0, to_save="./test.mp4"
):
    """
    Pass in data for single video, recons is recons frames, scores is x_std or x_mean etc.
    Threshold is RRE, mean, etc..
    """
    import matplotlib.gridspec as gridspec

    Writer = animation.writers["pillow"]
    writer = Writer(fps=15, metadata=dict(artist="Me"), bitrate=1800)
    gs = gridspec.GridSpec(2, 2, height_ratios=[2, 1])

    ht, wd = 64, 64

    eps = 0.0001
    # setup figure
    # fig = plt.figure()
    fig, ((ax1, ax3)) = plt.subplots(1, 2, figsize=(6, 6))

    ax1.axis("off")
    ax3.axis("off")
    # ax1=fig.add_subplot(2,2,1)

    ax1 = fig.add_subplot(gs[0, 0])
    ax1.set_title("Original")
    ax1.set_xticks([])
    ax1.set_yticks([])

    # ax2=fig.add_subplot(gs[-1,0])
    ax2 = fig.add_subplot(gs[1, :])

    # ax2.set_yticks([])
    # ax2.set_xticks([])
    ax2.set_ylabel("Score")
    ax2.set_xlabel("Frame")

    if threshold != 0:
        ax2.axhline(y=threshold, color="r", linestyle="dashed", label="RRE")
        ax2.legend()

    # ax3=fig.add_subplot(2,2,2)
    ax3 = fig.add_subplot(gs[0, 1])
    ax3.set_title("Reconstruction")
    ax3.set_xticks([])
    ax3.set_yticks([])

    # set up list of images for animation
    ims = []

    for time in range(1, len(testfall) - (win_len - 1) - 1):
        im1 = ax1.imshow(testfall[time].reshape(ht, wd), cmap="gray", aspect="equal")
        figure = recons[time].reshape(ht, wd)
        im2 = ax3.imshow(figure, cmap="gray", aspect="equal")
        # print(im1.shape)
        # print(im2.shape)

        # print("time={} mse={} std={}".format(time,mse_difficult[time],std))
        if time > 0:

            scores_curr = scores[0:time]

            fall_pts_idx = np.argwhere(scores_curr > threshold)
            nonfall_pts_idx = np.argwhere(scores_curr <= threshold)

            fall_pts = scores_curr[fall_pts_idx]
            nonfall_pts = scores_curr[nonfall_pts_idx]

            if fall_pts_idx.shape[0] > 0:
                # pass
                (plot_r,) = ax2.plot(fall_pts_idx, fall_pts, "r.")
                (plot,) = ax2.plot(nonfall_pts_idx, nonfall_pts, "b.")
            else:

                (plot,) = ax2.plot(scores_curr, "b.")

        else:
            (plot,) = ax2.plot(scores[0], "b.")
            (plot_r,) = ax2.plot(scores[0], "b.")

        ims.append([im1, plot, im2, plot_r])  # list of ims

    # run animation
    ani = animation.ArtistAnimation(fig, ims, interval=40, repeat=False)
    # plt.tight_layout()
    #plt.show()
    # gs.tight_layout(fig)
    ani.save(to_save)

    ani.event_source.stop()
    del ani
    plt.close()
    # plt.show()
    # return ani

def create_pytorch_dataset(name, dset, path, window_len, fair_compairson, stride):
    falls = []
    adl = []
    if fair_compairson == True:
        shared_adl_vids = np.loadtxt('shared_adl_vids.txt').astype(int)
        shared_fall_vids = np.loadtxt('shared_fall_vids.txt').astype(int)
        print(shared_fall_vids)
        # create list of all fall and nonfall folders
        for (root, dirs, files) in os.walk("F:/{}/Fall".format(dset)):
            for dir in dirs:
                x = re.findall('[0-9]+', dir)[0]
                if int(x) in shared_fall_vids:
                    falls.append(dir)

        for (root, dirs, files) in os.walk("F:/{}/NonFall".format(dset)):
            for dir in dirs:
                x = re.findall('[0-9]+', dir)[0]
                if int(x) in shared_adl_vids:
                    adl.append(dir)
        print(falls)
        print(adl)
    elif fair_compairson == False:
        # create list of all fall and nonfall folders
        for (root, dirs, files) in os.walk("F:/{}/Fall".format(dset)):
            if len(dirs) > 0:
                falls.extend(dirs)
        for (root, dirs, files) in os.walk("F:/{}/NonFall".format(dset)):
            if len(dirs) > 0:
                adl.extend(dirs)
        print(falls)
        print(adl)

    x_data_fall = []
    y_data_fall = []
    x_data_adl = []
    y_data_adl = []
    x_info_fall = []
    x_info_adl = []

    # path = "processed_data\data_set-{}-imgdim64x64.h5".format(name)

    # load in images of falls
    with h5py.File(path, "r") as hf:
        data_dict = hf["{}/Processed/Split_by_video".format(name)]
        # print(data_dict.keys())
        for Fall_name in falls:
            try:
                vid_total = data_dict[Fall_name]["Data"][:]
                if len(vid_total) < 100:
                    continue
                x_data_fall.append(vid_total)
                x_info_fall.append(Fall_name) #[4:]
                labels_total = data_dict[Fall_name]["Labels"][:]
                y_data_fall.append(labels_total)
            except:
                print("Skipped", Fall_name)
        for adl_name in adl:
            try:
                vid_total = data_dict[adl_name]["Data"][:]
                if len(vid_total) < 100:
                    continue
                x_data_adl.append(vid_total)
                x_info_adl.append(adl_name) #[7:]
                labels_total = data_dict[adl_name]["Labels"][:]
                y_data_adl.append(labels_total)
            except:
                print("Skipped", Fall_name)

    print(len(x_data_fall))
    print(len(y_data_fall))
    print(len(x_data_adl))
    print(len(y_data_adl))
    # get alll falls in data set
    print(x_info_adl)
    print(x_info_fall)
    # get matching day/night label from falls
    labels_dir = "F:/{}/".format(dset) + "Labels.csv"
    my_data = pd.read_csv(labels_dir)
    # sorting by first name
    my_data.sort_values("Video", inplace=True)
    my_data.drop_duplicates(subset="Video", keep="first", inplace=True)
    print(my_data.head())

    # pdb.set_trace()
    # %%    temp_df = my_data.loc[my_data["Video"] == int(fall), "ToD"]

    # ----------------------------------------------------------------------------
    # *** PREPARING DATASET LOADER ***
    # ----------------------------------------------------------------------------

    # 1) Need a ADL loader and a Fall Loader

    class Dataset(data.Dataset):
        "Characterizes a dataset for PyTorch"

        def __init__(self, labels, data, window):
            "Initialization"
            self.labels = labels
            self.data = data
            self.window = window

        def __len__(self):
            "Denotes the total number of samples"
            return len(self.data)

        def __getitem__(self, index):
            "Generates one sample of data"
            # prepare lists to dynamically fill with windows
            X_list = []
            Y_list = []
            # load a single video to chop up into windows
            ind_vid = self.data[index]
            ind_label = self.labels[index]
            # loop through each frame of the video (stopping window length short)
            for i in range(len(ind_vid) - self.window):
                # select the current window of the video
                X = ind_vid[i : i + self.window]
                y = ind_label[i : i + self.window]
                # add the current window the list of windows
                X_list.append(X)
                Y_list.append(y)
            # convert lists into arrays with proper size
            X = np.vstack(X_list)
            X = np.reshape(X_list, (len(ind_vid) - self.window, self.window, 64, 64))
            y = np.vstack(Y_list).T
            # X should be (window-length, 64, 64, # of windows w/in video) array
            # ex. (8, 64, 64, 192) for a 200 frame video and window size of 8
            # y is array (8, # of windows w/in video)
            return X, y

    Test_Dataset = Dataset(y_data_fall, x_data_fall, window=window_len)
    test_dataloader = data.DataLoader(Test_Dataset, batch_size=1)

    Train_Dataset = Dataset(y_data_adl, x_data_adl, window=window_len)
    train_dataloader = data.DataLoader(Train_Dataset, batch_size=1)

    return(Test_Dataset, test_dataloader, Train_Dataset, train_dataloader)

def create_multimodal_pytorch_dataset(name, dset, path, window_len, stride):

    def load_data_set(dset, name, path):
        falls = []
        adl = []
        print(dset)
        print(name)
        print(path)
        # create list of all fall and nonfall folders
        for (root, dirs, files) in os.walk("F:/{}/Fall".format(dset)):
            if len(dirs) > 0:
                falls.extend(dirs)
        for (root, dirs, files) in os.walk("F:/{}/NonFall".format(dset)):
            if len(dirs) > 0:
                adl.extend(dirs)

        x_data_fall = []
        y_data_fall = []
        x_data_adl = []
        y_data_adl = []
        x_info_fall = []
        x_info_adl = []

        # load in images of falls
        with h5py.File(path, "r") as hf:
            data_dict = hf["{}/Processed/Split_by_video".format(name)]
            # print(data_dict.keys())
            for Fall_name in falls:
                try:
                    vid_total = data_dict[Fall_name]["Data"][:]
                    if len(vid_total) < 100:
                        continue
                    x_data_fall.append(vid_total)
                    x_info_fall.append(Fall_name[4:])
                    labels_total = data_dict[Fall_name]["Labels"][:]
                    y_data_fall.append(labels_total)
                except:
                    print("Skipped", Fall_name)
            for adl_name in adl:
                try:
                    vid_total = data_dict[adl_name]["Data"][:]
                    if len(vid_total) < 100:
                        continue
                    x_data_adl.append(vid_total)
                    x_info_adl.append(adl_name[7:])
                    labels_total = data_dict[adl_name]["Labels"][:]
                    y_data_adl.append(labels_total)
                except:
                    print("Skipped", adl_name)

        print(len(x_data_fall))
        print(len(y_data_fall))
        print(len(x_data_adl))
        print(len(y_data_adl))
        '''
        # get matching day/night label from falls
        labels_dir = "F:/{}/".format(dset) + "Labels.csv"
        my_data = pd.read_csv(labels_dir)
        # sorting by first name
        my_data.sort_values("Video", inplace=True)
        my_data.drop_duplicates(subset="Video", keep="first", inplace=True)
        x_info_fall_ToD = []
        for fall in x_info_fall:
            temp_df = my_data.loc[my_data["Video"] == int(fall), "ToD"]
            x_info_fall_ToD.append(temp_df.values.astype(int))
        x_info_fall_ToD = np.vstack(x_info_fall_ToD)
        np.savetxt(
            "results/Edits/{}/night_time_{}.csv".format(str(name) + str(name), start_time),
            x_info_fall_ToD,
            delimiter=",",
        )
        '''

        return x_data_fall, y_data_fall, x_data_adl, y_data_adl

     # TODO: Update this to dynaically take any list of inputs, irregardless of length
    class Dataset(data.Dataset):
        "Characterizes a dataset for PyTorch"

        def __init__(self, data, labels, window):
            "Initialization"
            self.data = data
            self.labels = labels
            self.window = window

        def __len__(self):
            "Denotes the total number of samples"
            return len(self.data[0])

        def __getitem__(self, index):
            #defining a function at start

            def create_window(ind_vid):
                X_list =[]
                Y_list = []
                for i in range(0, len(ind_vid) - self.window):
                    # select the current window of the video
                    x = ind_vid[i : i + self.window]
                    y = ind_label[i : i + self.window]
                    # add the current window the list of windows
                    X_list.append(x)
                    Y_list.append(y)
                return X_list, Y_list

            "Generates one sample of data"
            # prepare lists to dynamically fill with windows
            X_all_mod = []
            y_all_mod = []
            # load a single modality to work on
            for i in range(len(self.data)):
                current_mod = self.data[i]
                current_label = self.labels[i]
                # load a single video to create windows of the data
                ind_vid = current_mod[index]
                ind_label = current_label[index]
                X_list, Y_list = create_window(ind_vid)
                X = np.vstack(X_list)
                X = np.reshape(X_list, (len(ind_vid) - self.window, self.window, 64, 64))
                y = np.vstack(Y_list).T
                X_all_mod.append(X)
                y_all_mod.append(y)

                # X should be (window-length, 64, 64, # of windows w/in video) array
                # ex. (8, 64, 64, 192) for a 200 frame video and window size of 8
                # y is array (8, # of windows w/in video)
                # this is all stored in a list with each element being a different modality
            return X_all_mod, y_all_mod

    y_data_fall = []
    x_data_fall = []
    y_data_adl = []
    x_data_adl = []

    print(dset[0])
    print(dset[1])
    print(name[0])
    print(name[1])
    for i in range(len(dset)):
        path = "H5Data\data_set-{}-imgdim64x64.h5".format(name[i])
        print(path)
        x1, y1, x2, y2 = load_data_set(dset[i], name[i], path)
        # where x1 and y1 contain the data and labels for videos with falls
        # and that x2 and y2 have videos with ADL activities for training
        x_data_fall.append(x1)
        y_data_fall.append(y1)
        x_data_adl.append(x2)
        y_data_adl.append(y2)
        # print(i)

    Test_Dataset = Dataset(
        x_data_fall, y_data_fall, window=window_len
    )
    test_dataloader = data.DataLoader(Test_Dataset, batch_size=1)

    Train_Dataset = Dataset(
        x_data_adl, y_data_adl, window=window_len
    )
    train_dataloader = data.DataLoader(Train_Dataset, batch_size=1)

    return(Test_Dataset, test_dataloader, Train_Dataset, train_dataloader)


import os
import sys
import cv2
import json
import numpy as np
import urllib.request

# load the labels
coco91_labels = ['__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
    'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'N/A', 'stop sign',
    'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
    'elephant', 'bear', 'zebra', 'giraffe', 'N/A', 'backpack', 'umbrella', 'N/A', 'N/A',
    'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball',
    'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
    'bottle', 'N/A', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
    'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza',
    'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'N/A', 'dining table',
    'N/A', 'N/A', 'toilet', 'N/A', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',
    'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'N/A', 'book',
    'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
# this will help us create a different color for each class
coco91_colors = np.random.uniform(0, 255, size=(len(coco91_labels), 3))

def preprocess_imagenet(img, chw=True, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], img_size=(224,224), make_square=True):
    x = img.copy()
    h,w,c = x.shape
    if make_square:
        max_side = 0
        if h!=w: # squared - make it square to avoid distortions
            max_side = max(h,w)
            new_img = np.zeros((max_side,max_side,c), dtype=np.uint8)
            new_img[0:h, 0:w] = x[:]
            x = new_img
    x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)
    x = cv2.resize(x, img_size)
    x = (x / 255)
    x = (x - mean) / std # normalize
    if chw: x = x.transpose(2,0,1) # HWC to CHW
    x = np.expand_dims(x, 0).astype(np.float32)
    return x

def draw_boxes(boxes, classes, labels, image, img_size=300):
    """
    Draws the bounding box around a detected object.
    """
    h,w,c = image.shape
    max_side = max(h,w)
    fx,fy = max_side/img_size,max_side/img_size
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    for i, box in enumerate(boxes):
        box = (box.astype(np.float32) * (fx,fy,fx,fy)).astype(np.int32)
        #box += (0,-r,0,r)
        color = coco91_colors[labels[i]]
        cv2.rectangle(
            image,
            (int(box[0]), int(box[1])),
            (int(box[2]), int(box[3])),
            color, w//150
        )
        cv2.putText(image, classes[i], (int(box[0]), int(box[1]-5)),
                    cv2.FONT_HERSHEY_SIMPLEX, int(0.5*fx), color, int(1*fx),
                    lineType=cv2.LINE_AA)
    return image

def load_sample_imgA():
    if not os.path.exists("goldfish.jpg"):
        urllib.request.urlretrieve("https://images.unsplash.com/photo-1579161256825-57ba3094f057", "goldfish.jpg")
    return cv2.imread("goldfish.jpg")

def load_sample_imgB():
    if not os.path.exists("zidane.jpg"):
        urllib.request.urlretrieve("https://ultralytics.com/images/zidane.jpg", "zidane.jpg")
    return cv2.imread("zidane.jpg")

def load_sample_imgC():
    if not os.path.exists("man_walking.jpg"):
        urllib.request.urlretrieve("https://images.unsplash.com/photo-1611324204543-ecc01e953173", "man_walking.jpg")
    return cv2.imread("man_walking.jpg")

def load_sample_imgD():
    if not os.path.exists("signs.jpg"):
        urllib.request.urlretrieve("https://cdn-01.media-brady.com/store/stca/media/catalog/product/r/e/reflective-warning-signs-deer-crossing-ac0568-lg.jpg", "signs.jpg")
    return cv2.imread("signs.jpg")

def load_sample_imgE():
    if not os.path.exists("text.jpg"):
        urllib.request.urlretrieve("https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg", "text.jpg")
    return cv2.imread("text.jpg")

def load_imagenet1k_labels():
    # Download the labels for Imagenet 1k
    if not os.path.exists("imagenet1k.json"):
        urllib.request.urlretrieve(
            "https://raw.githubusercontent.com/anishathalye/imagenet-simple-labels/master/imagenet-simple-labels.json",
            "imagenet1k.json"
        )
    return json.loads(open('imagenet1k.json', 'r').read())

def load_coco80_labels():
    # load the labels
    if not os.path.exists("coco80.json"):
        urllib.request.urlretrieve(
            "https://raw.githubusercontent.com/amikelive/coco-labels/master/coco-labels-2014_2017.txt",
            "coco80.labels"
        )
    labels = open('coco80.labels', 'r').readlines()
    return [l.strip() for l in labels]


import time
import datetime
import inspect
import logging

def fstr(fstring_text, locals, globals=None):
    """
    Dynamically evaluate the provided fstring_text

    Sample usage:
        format_str = "{i}*{i}={i*i}"
        i = 2
        fstr(format_str, locals()) # "2*2=4"
        i = 4
        fstr(format_str, locals()) # "4*4=16"
        fstr(format_str, {"i": 12}) # "10*10=100"
    """
    locals = locals or {}
    globals = globals or {}
    ret_val = eval(f'f"{fstring_text}"', locals, globals)
    return ret_val

class timelogged(object):
    """
    Decorator class for logging function start, completion, and elapsed time.
    """

    def __init__(
        self,
        desc_text="'{desc_detail}' call to {fn.__name__}()",
        desc_detail="",
        start_msg="Выполняется {desc_text}...",
        success_msg="Готово: {desc_text}  {elapsed}",
        log_fn=logging.info,
        **addl_kwargs,
    ):
        """ All arguments optional """
        self.context = addl_kwargs.copy()  # start with addl. args
        self.context.update(locals())  # merge all constructor args
        self.context["elapsed"] = None
        self.context["start"] = time.time()

    def re_eval(self, context_key: str):
        """ Evaluate the f-string in self.context[context_key], store back the result """
        self.context[context_key] = fstr(self.context[context_key], locals=self.context)

    def elapsed_str(self):
        """ Return a formatted string, e.g. '(HH:MM:SS elapsed)' """
        seconds = time.time() - self.context["start"]
        delta = datetime.timedelta(seconds=seconds)
        str_delta = ""

        seconds = delta.total_seconds()
        hours = int(seconds // 3600)
        minutes = int((seconds % 3600) // 60)
        seconds = int(seconds % 60)

        if hours:
            str_delta += str(minutes) + "h "

        if minutes:
            str_delta += str(minutes) + "m "

        if seconds:
            str_delta += str(seconds) + "s "

        milliseconds = round(delta.microseconds / 1000)
        str_delta += str(milliseconds) + "ms"

        return "(за {})".format(str_delta)

    def __call__(self, fn):
        """ Call the decorated function """

        def wrapped_fn(*args, **kwargs):
            """
            The decorated function definition. Note that the log needs access to
            all passed arguments to the decorator, as well as all of the function's
            native args in a dictionary, even if args are not provided by keyword.
            If start_msg is None or success_msg is None, those log entries are skipped.
            """
            self.context["fn"] = fn
            fn_arg_names = inspect.getfullargspec(fn).args
            for x, arg_value in enumerate(args, 0):
                self.context[fn_arg_names[x]] = arg_value
            self.context.update(kwargs)
            desc_detail_fn = None
            log_fn = self.context["log_fn"]
            # If desc_detail is callable, evaluate dynamically (both before and after)
            if callable(self.context["desc_detail"]):
                desc_detail_fn = self.context["desc_detail"]
                self.context["desc_detail"] = desc_detail_fn()

            # Re-evaluate any decorator args which are fstrings
            self.re_eval("desc_detail")
            self.re_eval("desc_text")
            # Remove 'desc_detail' if blank or unused
            self.context["desc_text"] = self.context["desc_text"].replace("'' ", "")
            self.re_eval("start_msg")
            if self.context["start_msg"]:
                # log the start of execution
                log_fn(self.context["start_msg"])
            ret_val = fn(*args, **kwargs)
            if desc_detail_fn:
                # If desc_detail callable, then reevaluate
                self.context["desc_detail"] = desc_detail_fn()
            self.context["elapsed"] = self.elapsed_str()
            # log the end of execution
            log_fn(fstr(self.context["success_msg"], locals=self.context))
            return ret_val

        return wrapped_fn


'''

'''
import pdb
import argparse
import os
import sys
import utils.matching_normalization as mt_norm

sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("--mci_stationary_data_path", type=str, default='stationary_data/stationary_dataset_mci.csv')
parser.add_argument("--non_mci_stationary_data_path", type=str, default='stationary_data/stationary_dataset_nonmci.csv')

parser.add_argument("--mci_metadata_path", type=str, default='intermediate_files/mci_metadata.csv')
parser.add_argument("--non_mci_metadata_path", type=str, default='intermediate_files/nonmci_metadata.csv')

parser.add_argument("--matching", type=int, default=1, choices=[0,1])
parser.add_argument("--case_control_ratio", type=int, default=1)

parser.add_argument("--test_ratio", type=int, default=0.3)

output_path = mt_norm.matching(parser.parse_args().mci_stationary_data_path
				,parser.parse_args().non_mci_stationary_data_path
				,parser.parse_args().mci_metadata_path
				,parser.parse_args().non_mci_metadata_path
				,parser.parse_args().case_control_ratio
				,parser.parse_args().matching
				)

mt_norm.normalization(output_path
					,parser.parse_args().test_ratio
					)


# coding=utf-8
# __author__ = 'xhm'
import sys
import librosa
import librosa.display
import numpy as np

def get_para():
    return sys.argv[1], sys.argv[2]

def get_music_name(name_path):
    split = name_path.split('/')
    music_name = split[-1].strip()
    return music_name

def mel_data(folder_path, txt_path):
    file = open(txt_path, 'r')
    lines = file.readlines()
    for line in lines:
        y, sr = librosa.load(path=line.strip())
        s = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=40, n_fft=3072, hop_length=768, fmax=8000)
        re = librosa.power_to_db(s)
        music_name = get_music_name(line)
        save_path = folder_path + '/' + music_name
        np.save(save_path, re)
        print("Processing " + line.strip() + "...... done")
    file.close()

def main(argv=None):
    folder_path, txt_path = get_para()
    mel_data(folder_path, txt_path)
    print("Preprocessing and Extracting Features........All done")
if __name__ == '__main__':
    main()


# coding=utf-8
#-------------------------------------------------------------------------------

#Author: Alexandre Manhaes Savio <alexsavio@gmail.com>
#Grupo de Inteligencia Computational <www.ehu.es/ccwintco>
#Universidad del Pais Vasco UPV/EHU
#
#2013, Alexandre Manhaes Savio
#Use this at your own risk!
#-------------------------------------------------------------------------------

import os
import sys
import shelve
import logging as log

import numpy as np
import nibabel as nib
from sklearn.grid_search import ParameterGrid
from sklearn.preprocessing import StandardScaler

from .io import load_data
from ..files.names import get_extension
from ..storage import save_varlist_to_shelve

def perform_classification (subjsf, labelsf, outfile, datadir, maskf, clfmethod, fsmethod1,
                            fsmethod2, prefsmethod, prefsthr, cvmethod, thrmethod, stratified, stddize, n_cpus):
    """
    @param subjsf:
    @param labelsf:
    @param outfile:
    @param datadir:
    @param maskf:
    @param clfmethod:
    @param fsmethod1:
    @param fsmethod2:
    @param prefsmethod:
    @param prefsthr:
    @param cvmethod:
    @param thrmethod:
    @param stratified:
    @param stddize:
    @param n_cpus:
    @return:
    """
    #first list of variables to be saved
    shelfnames = ['subjsf', 'maskf', 'datadir', 'prefsmethod', 'prefsthr', 'clfmethod', 'fsmethod1', 'fsmethod2', 'stddize']

    scores = None
    if get_extension(subjsf) != '.npy':
        X, y, scores, imgsiz, mask, indices = load_data(subjsf, datadir, maskf, labelsf)
    else:
        X = np.load(subjsf)
        y = np.loadtxt(labelsf)
        mask = nib.load(maskf).get_data()
        #indices = np.where(mask > 0)

    #for COBRE ONLY, removing last two subjects:
    X = X[:-2, :]
    y = y[:-2]

    preds, probs, best_pars, presels, cv, \
    importance, scores, y, truth = extract_and_classify(X, y, scores, prefsmethod, prefsthr, fsmethod1, fsmethod2,
                                                        clfmethod, cvmethod, stratified, stddize, thrmethod, n_cpus)

    #save results with pickle
    log.info('Saving results in ' + outfile)

    #extend list of variables to be saved
    shelfnames.extend([ 'y', 'mask', 'indices', 'cv', 'importance', 'preds', 'probs', 'scores', 'best_pars', 'truth', 'presels'])

    shelfvars = []
    for v in shelfnames:
        shelfvars.append(eval(v))

    save_varlist_to_shelve(outfile + '.pyshelf', shelfnames, shelfvars)

    return cv, truth, preds, probs, presels

def pre_featsel(X, y, method, thr=95, dist_function=None, thr_method='robust'):
    '''
    INPUT
    X             : data ([n_samps x n_feats] matrix)
    y             : class labels
    method        : distance measure: 'pearson', 'bhattacharyya', 'welcht', ''
                    if method == '', will use dist_function
    thr           : percentile distance threshold
    dist_function :
    thr_method    : method for thresholding: 'none', 'robust', 'ranking'

    OUTPUT
    m          : distance measure (thresholded or not)
    '''

    #pre feature selection, measuring distances
    #Pearson correlation
    if method == 'pearson':
        au.log.info ('Calculating Pearson correlation')
        m = np.abs(pearson_correlation(X, y))

    #Bhattacharyya distance
    elif method == 'bhattacharyya':
        au.log.info ('Calculating Bhattacharyya distance')
        m = bhattacharyya_dist (X, y)

    #Welch's t-test
    elif method == 'welcht':
        au.log.info ("Calculating Welch's t-test")
        m = welch_ttest (X, y)

    elif method == '':
        au.log.info ("Calculating distance between data and class labels")
        #http://docs.scipy.org/doc/scipy/reference/spatial.distance.html
        m = distance_computation(X, y, dist_function)

    #if all distance values are 0
    if not m.any():
        au.log.info("No differences between groups have been found. Are you sure you want to continue?")
        return m

    #threshold data
    if thr_method != 'none':
        if thr_method == 'robust':
            mt = au.robust_range_threshold (m, thr)
        elif thr_method == 'percentile':
            mt = au.percentile_threshold (m, thr)
        elif thr_method == 'rank':
            mt = au.rank_threshold (m, thr)

        return mt

    return m

def extract_and_classify (X, y, scores, prefsmethod, prefsthr, fsmethod1, fsmethod2,
                          clfmethod, cvmethod, stratified, stddize,
                          thrmethod='robust', n_cpus=1, gs_scoring='accuracy'):
    '''
    Parameters
    ----------
    X:

    y:

    scores:

    prefsmethod:

    prefsthr:

    fsmethod1:

    fsmethod2:

    clfmethod:

    cvmethod:

    stratified: bool

    stddize: bool

    thrmethod: string

    n_cpus: int

    gs_scoring: string, callable or None, optional, default: None
        Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision',
        'f1', 'log_loss', 'mean_squared_error', 'precision', 'r2', 'recall', 'roc_auc']
        See GridSearchCV docs for further details.

    Returns
    -------
    preds, probs, best_pars, presels, cv, importance, scores, y, truth

    '''

    #classifiers
    #cgrid    = [10**-3, 10**-2, 10**-1, 10**0, 10**1, 10**2]
    #if nclass
    #perfmeas = ['Accuracy', 'Precision', 'Recall', 'F1', 'PRBEP', 'ROCArea', 'AvgPrec']

    #defining parameters for classifiers
    n_class    = len(np.unique(y))
    n_subjs    = X.shape[0]
    n_feats    = X.shape[1]
    n_selfeats = np.min(n_feats, int(np.floor(n_subjs*0.06)))

    cv = get_cv_method(y, cvmethod, stratified)

    presels    = {}
    preds      = {}
    probs      = {}
    truth      = {}
    best_pars  = {}
    importance = {}
    fc = 0
    for train, test in cv:
        au.log.info('Processing fold ' + str(fc))

        #data cv separation
        X_train, X_test, y_train, y_test = X[train,:], X[test,:], y[train], y[test]

        #scaling
        #if clfmethod == 'linearsvc' or clfmethod == 'onevsonesvc':
        if stddize:
            log.info('Standardizing data')
            scaler = StandardScaler()
            X_train = scaler.fit_transform(X_train)
            X_test  = scaler.transform    (X_test)
            #[X_train, dmin, dmax] = au.rescale (X_train, scale_min, scale_max)
            #[X_test,  emin, emax] = au.rescale (X_test,  scale_min, scale_max, dmin, dmax)

        #PRE feature selection
        if prefsmethod != 'none':
            #sc_train = scores[train]
            presels[fc] = pre_featsel (X_train, y_train, prefsmethod, prefsthr, thrmethod)
            if not presels[fc].any():
                log.info('No feature survived the ' + prefsmethod + '(' + thrmethod + ': '+ str(prefsthr) + ')' + ' feature selection.')
                continue

            X_train = X_train[:, presels[fc] > 0]
            X_test  = X_test [:, presels[fc] > 0]

        pipe, params = get_pipeline(fsmethod1, fsmethod2, clfmethod, n_subjs, n_feats, n_cpus)

        #creating grid search
        gs = GridSearchCV(pipe, params, n_jobs=n_cpus, verbose=1, scoring=gs_scoring)

        #do it
        log.info('Running grid search')
        gs.fit(X_train, y_train)

        log.info('Predicting on test set')

        #predictions, feature importances and best parameters
        preds     [fc] = gs.predict(X_test)
        truth     [fc] = y_test
        best_pars [fc] = gs.best_params_

        if hasattr(gs.best_estimator_, 'support_vectors_'):
            importance[fc] = gs.best_estimator_.support_vectors_
        elif hasattr(gs.best_estimator_, 'feature_importances_'):
            importance[fc] = gs.best_estimator_.feature_importances_

        if hasattr(gs.estimator, 'predict_proba'):
            try:
                probs [fc] = gs.predict_proba(X_test)
            except:
                probs [fc] = []

        #hello user
        au.log.info( 'Result: ' + str(y_test) + ' classified as ' + str(preds[fc]))

        fc += 1

    return preds, probs, best_pars, presels, cv, importance, scores, y, truth

def do_experiment_paramgrid (wd, masks, dataf, labelsf, param_grid, thrmethod, stratified, stddize, cvfolds, n_cpus):

    results = {}
    for j in list(ParameterGrid(param_grid)):

        subjsf    = j['subjsf']
        cl        = j['cl']
        prefs     = j['prefs']
        prefs_thr = j['prefs_thr']
        fs1       = j['fs1']
        fs2       = j['fs2']

        maskf = masks[dataf.index(subjsf)]

        of = os.path.join(wd, au.remove_ext(subjsf))
        if prefs != 'none': of += '_' + prefs + '_' + str(prefs_thr)
        if fs1   != 'none': of += '_' + fs1
        of += '_' + cl

        if   cl == 'rf': stddize = False
        else           : stddize = True

        print("Running " + prefs + " " + str(prefs_thr) + " " + fs1 + "_" + str(j))

        if not os.path.exists(of + '.pyshelf'):
            cv, truth, preds, probs, presels = do_classification (subjsf, labelsf, of, wd, maskf, cl, fs1, fs2, prefs, prefs_thr, cvfolds, thrmethod, stratified, stddize, n_cpus)
        else:
            print('Previously done.')
            print('Loading ' + of)
            res = shelve.open(of + '.pyshelf')
            cv, truth, preds, probs, presels = res['cv'], res['truth'], res['preds'], res['probs'], res['presels']

        #save results
        #metrics[c, :] = np.array([acc, sens, spec, prec, f1, roc_auc])
        metrics = aizc.get_cv_classification_metrics (truth, preds, probs)

        j['y_pred']  = preds
        j['y_true']  = truth

        j['metrics'] = metrics
        j['presels'] = presels

        if subjsf not in results:
            results[subjsf] = []

        results[subjsf].append(Result(**j))

    return results

##==============================================================================
##execution script
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')

import do_cobre_classification as cb
results = cb.do_experiment(2)

cb.plot_all_results (results)

'''
def do_experiment(n_cpus=1):

    au.setup_logger(2)

    temporal_filtering     = [True, False]
    global_nuis_correction = [True, False]
    preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

    results = {}
    for j in list(ParameterGrid(preprocess_grid)):
        wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(**j)

        stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()
        param_grid    = {'subjsf': dataf, 'cl': clf_methods, 'prefs' : prefs_methods, 'prefs_thr': prefs_thrs, 'fs1' : ['none'], 'fs2' : ['none']}

        #loop will load all classification results files (or classify again) and calculate performance metrics
        results[str(j)] = do_experiment_paramgrid(wd, masks, dataf, labelsf, param_grid, thrmethod, stratified, stddize, cvfolds, n_cpus)

        #plot_results (results[str(j)], wd, masks, dataf, param_grid, stratified, stddize, cvfolds)

    return results

##==============================================================================
##execution script
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')

import do_cobre_classification as cb
results = cb.do_experiment(2)

sum_tabs = cb.print_all_summary_tables (results)

'''
def print_all_summary_tables(results):
    from do_cobre_featsets import get_filepaths

    temporal_filtering     = [True, False]
    global_nuis_correction = [True, False]
    preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

    stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()

    sum_tab = {}
    for j in list(ParameterGrid(preprocess_grid)):

        wd, xd, dd, labelsf, phenof, subjsf, masks, dilmasks, templates, pipe = get_filepaths(**j)

        print(str(j))
        print('Doing pipeline ' + pipe + '\n')

        sum_tab[str(j)] = print_summary_table(results[str(j)], wd, masks, subjsf, prefs_methods, prefs_thrs, clf_methods)

        #prefs_method

    return sum_tab

##==============================================================================
def print_summary_table (results, wd, masks, subjsf, prefs_methods, prefs_thrs, clf_methods):

    ##SHOW FINAL RESULTS
    datatype = [
    'ALFF',
    'fALFF',
    'ReHo',
    'VMHC',
    'VMHC stat',
    ]

    sum_tab = {}
    for fi, f in enumerate(subjsf):

        f = subjsf[fi]
        #print f
        resf = results[f]
        #print len(resf)
        d = datatype[fi]
        sum_tab[d] = {}

        for p in clf_methods:
            #print p
            resfp = aizc.filter_objlist (resf, 'cl', p)

            #print (p + '_' + os.path.basename(f))

            sum_tab[d][p] = {}
            crow = ''
            i = 0
            for c in prefs_methods:
                sum_tab[d][p][c] = {}

                resfpc = aizc.filter_objlist (resfp, 'prefs', c)

                i += 1

                # getting accuracies
                signfs = []
                means, varis = [], []
                m_idx = 0 #metric index
                for t in prefs_thrs:
                    resfpct = aizc.filter_objlist (resfpc, 'prefs_thr', t)[0]

                    #metrics[i, :] = np.array([acc, sens, spec, prec, f1, roc_auc])
                    metrs = np.array(resfpct.metrics)

                    means.append(metrs.mean(axis=0)[m_idx])
                    varis.append(metrs.var (axis=0)[m_idx])

                    signfs.append(aizc.get_cv_significance(resfpct.y_true, resfpct.y_pred))

                max_means = np.max(means)
                max_varis = np.max(varis)

                row   = "%.2f (%.2f)" % (max_means, max_varis)
                crow += '\t' + row
                #row = "%.2f (%.2f)" % (max_means[m_idx], max_varis[m_idx])
                #print (d + ' ' + p + ' ' + c + ' ' + row)
                sum_tab[d][p][c] = row

            print (d + ' ' + p + ' \t (' + str(prefs_methods) + '): ' + crow)

    return sum_tab

##==============================================================================
def get_localizations (X, y, cv, maskf, presels, sv):

    mask, hdr, aff = au.get_nii_data(maskf)
    maskidx = np.array(np.where(mask > 0))
    hdr.set_data_dtype(np.dtype(np.float))

    my_presels  = np.zeros_like(presels[0])
    my_svs      = np.zeros_like(mask)
    my_svs_done = False

    #unmasking method found in:
    #http://nisl.github.io/auto_examples/plot_ica_resting_state.html
    from nisl.io import NiftiMasker

    k = 0
    cv.n = X.shape[0]
    for train,test in cv:

        X_train, y_train = X[train,:], y[train]

        preselsvol = np.zeros_like (mask, dtype=np.float)
        preselsvol[tuple(maskidx)] = presels[k] > 0
        preselsnii = au.save_nibabel ('', preselsvol, aff, hdr)

        my_presels += presels[k] > 0

        if len(sv) > 0:
            try:
                nifti_masker = NiftiMasker(mask=preselsnii)
                nifti_masker.fit (X_train[:,presels[k]>0], y_train)
                niimg = nifti_masker.inverse_transform(sv[k][0])
                #X_masked = nifti_masker.fit_transform (X_train[:,presels[k]>0], y_train) #,y_train, target_affine=aff, target_shape=hdr.get_data_shape()
                #niimg = nifti_masker.inverse_transform(sv[0][0])
                #act = np.ma.masked_array(niimg.get_data(), niimg.get_data() == 0)

                my_svs += niimg.get_data()
                my_svs_done = True
            except:
                pass

        k += 1

    my_presels /= cv.n_folds
    my_svs     /= cv.n_folds

    prelocs = np.zeros_like (mask, dtype=np.float)
    prelocs[tuple(maskidx)] = my_presels

    return prelocs, my_svs, my_svs_done

##==============================================================================
##Calculating and saving localization masks
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_cobre_classification as cb
from sklearn.grid_search import ParameterGrid

volaxis=2
save_fig=True
overwrite=True

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

for j in list(ParameterGrid(preprocess_grid)):
    cb.do_localization_images(j['temporal_filtering'], j['global_nuis_correction'],
                        save_fig=save_fig, volaxis=volaxis, overwrite=overwrite)
'''
def do_localization_images (temporal_filtering=True, global_nuis_correction=True,
                           save_fig=True, volaxis=0, overwrite=False):

    if save_fig:
        import matplotlib
        matplotlib.use('AGG')

    import pylab as plt
    from sklearn.grid_search import ParameterGrid

    sys.path.append('/home/alexandre/Dropbox/Documents/work/visualize_volume')
    import visualize_volume as vis

    #save_fig = True
    #temporal_filtering     = True
    #global_nuis_correction = False
    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    au.setup_logger(2, logfname=None)

    stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()
    param_grid    = {'subjsf': dataf, 'cl': clf_methods, 'prefs' : prefs_methods, 'prefs_thr': prefs_thrs, 'fs1' : ['none']}

    #loop will load all classification results dataf (or classify again) and calculate performance metrics
    results = {}
    for j in list(ParameterGrid(param_grid)):

        subjsf    = j['subjsf']
        cl        = j['cl']
        prefs     = j['prefs']
        prefs_thr = j['prefs_thr']
        #fs1 = j['fs1']
        #fs2 = j['fs2']

        of = os.path.join(wd, au.remove_ext(subjsf))
        if prefs != 'none': of += '_' + prefs + '_' + str(prefs_thr)
        #if fs1   != 'none': of += '_' + fs1
        #if fs2   != 'none': of += '_' + fs2
        #of += '_' + cl

        #checking if output files exist
        fig1name = of + '_presels_axis' + str(volaxis) + '.png' #presels are the same for all classifiers

        of = of + '_' + cl

        fig2name = of + '_sup_vecs_axis' + str(volaxis) + '.png'

        #overwrite?
        if not overwrite:
            if os.path.exists(fig1name) and os.path.exists(fig2name):
                print("File " + fig1name + " already exists, jumping to next.")
                continue

        print("Running " + prefs + " " + str(prefs_thr) + " " + str(j))

        #loading data file
        data_file = of + '.pyshelf'
        print('Loading ' + data_file)
        res = shelve.open(data_file)

        try:
            presels = res['presels']
            cv      = res['cv']
            sv      = res['importance']
        finally:
            res.close()

        #Loading data within cv in order to unmask feature_importances_
        X = np.load(subjsf)
        y = np.loadtxt(labelsf)

        fidx   = dataf.index(subjsf)
        maskf  = masks[fidx]
        tmpltf = templates[fidx]

        tmplt  = nib.load(tmpltf).get_data()

        #getting localizations from data
        presels_vol, supvecs_vol, supvecs_done = get_localizations (X, y, cv, maskf, presels, sv)

        #preselections figure
        fig1 = vis.show_many_slices(tmplt, presels_vol, volaxis=volaxis, vol2_colormap=plt.cm.hot, figtitle=of + '_presels')
        if save_fig:
            aizc.save_fig_to_png (fig1, fig1name, facecolor='white')
            #call_innercrop (xd, fig1name)
        else:
            input("Press Enter to continue...")

        #feature_importances figure
        if supvecs_done:
            fig2 = vis.show_many_slices(tmplt, supvecs_vol, volaxis=volaxis, vol2_colormap=plt.cm.hot, figtitle=of + '_sup_vecs')
            if save_fig:
                aizc.save_fig_to_png (fig2, fig2name, facecolor='white')
                #call_innercrop(xd, fig2name)
            else:
                input("Press Enter to continue...")

##==============================================================================
def call_innercrop(xd, figname):
    if os.access (os.path.join(xd,'innercrop'), os.X_OK):
        comm = os.path.join(xd,'innercrop') + ' -o white ' + figname + ' ' + figname
        print('Calling: ' + comm)
        au.sys_call(comm)
    else:
        print('Could not find innercrop for execution.')

##==============================================================================
def call_autoaq(xd, maskf, thresh, atlas, outfile):
    if os.access (os.path.join(xd,'autoaq'), os.X_OK):
        comm = os.path.join(xd,'autoaq') + ' -i ' + maskf + ' -t ' + str(thresh) + ' -a ' + '"' + atlas + '"' + ' -o ' + outfile
        print('Calling: ' + comm)
        au.exec_command (comm)
    else:
        print('Could not find autoaq for execution.')

##==============================================================================
def call_atlasquery(xd, maskf, atlas, outfile):
    if os.access ('/usr/bin/atlasquery', os.X_OK):
        comm = 'atlasquery' + ' -m ' + maskf + ' -a ' + '"' + atlas + '"' + ' > ' + outfile
        print('Calling: ' + comm)
        au.exec_command (comm)
    else:
        print('Could not find atlasquery for execution.')

##==============================================================================
def call_atlasquerpy(xd, maskf, atlas, outfile):
    import subprocess
    import numpy as np
    atlasquerpy = '/home/alexandre/Dropbox/Documents/work/atlasquerpy/atlasquerpy'
    if os.access (atlasquerpy, os.X_OK):
        comm = atlasquerpy + ' -t roiover -m ' + maskf + ' -a ' + '"' + atlas + '"' + ' > ' + outfile
        print('Calling: ' + comm)
        process = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        output, error = process.communicate()
        #subprocess.call (comm, shell=True)
        #run_cmd (comm)
        np.savetxt(outfile, np.array(output.split('\n')[:-1]), fmt='%s')
    else:
        print('Could not find atlasquery for execution.')

##==============================================================================
def get_resultfile_name (subjsf, cl, prefs, prefs_thr, fs1='none', fs2='none'):

    resultf = au.remove_ext(subjsf)
    if prefs != 'none': resultf += '_' + prefs + '_' + str(prefs_thr)
    if fs1   != 'none': resultf += '_' + fs1
    if fs2   != 'none': resultf += '_' + fs2
    resultf += '_' + cl
    resultf += '.pyshelf'

    return resultf

##==============================================================================
def get_data_from_shelf (resultfile_path):
    #loading data file
    res = shelve.open(resultfile_path)

    try:
        presels = res['presels']
        cv      = res['cv']
        sv      = res['importance']
    finally:
        res.close()

    return presels, cv, sv

##==============================================================================
def get_localizations_from_datashelf(resultfile_path, subjsf, labelsf, maskf):

    presels, cv, sv = get_data_from_shelf (resultfile_path)

    #Loading data within cv in order to unmask feature_importances_
    X = np.load(subjsf)
    y = np.loadtxt(labelsf)

    #getting localizations from data
    presels_vol, supvecs_vol, supvecs_done = get_localizations (X, y, cv, maskf, presels, sv)

    return presels_vol

##==============================================================================
##Calculating and saving localization masks
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_cobre_classification as cb

save_fig=True
overwrite=True

cb.do_tanimoto_comparison(save_fig=save_fig, overwrite=overwrite)
'''
def do_tanimoto_comparison (save_fig=True, overwrite=False):

    if save_fig:
        import matplotlib
        matplotlib.use('AGG')

    import pylab as plt
    from sklearn.grid_search import ParameterGrid

    sys.path.append('/home/alexandre/Dropbox/Documents/work/visualize_volume')

    temporal_filtering     = [True, False]
    global_nuis_correction = [True, False]
    preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

    fig = plt.figure('Feature Vectors Tanimoto Distance')
    c = 0

    for j in list(ParameterGrid(preprocess_grid)):

        temporal_filtering, global_nuis_correction = j['temporal_filtering'], j['global_nuis_correction']

        #save_fig = True
        #temporal_filtering     = True
        #global_nuis_correction = False
        wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

        au.setup_logger(2, logfname=None)

        stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()
        param_grid    = {'subjsf': dataf[0:3], 'cl': [clf_methods[0]]}

        prefs_thrs = [95]

        for prefs_m in prefs_methods:

            for prefs_thr in prefs_thrs:

                tani_mat, ticks = tanimoto_distances_matrix (param_grid, temporal_filtering, global_nuis_correction, prefs_m, prefs_thr)

                fname = 'tanimoto_distance_matrix_' + prefs_m + '_' + str(prefs_thr) + '.png'

                ax = plt.subplot(3,4,c)

                ax.imshow(tani_mat, cmap=plt.cm.jet, interpolation='nearest')

                axtit = ''
                if temporal_filtering and global_nuis_correction:
                    axtit += 'TPF-GSR'
                elif temporal_filtering and not global_nuis_correction:
                    axtit += 'TPF'
                elif not temporal_filtering and global_nuis_correction:
                    axtit += 'GSR'

                # Move left and bottom spines outward by 10 points
                ax.spines['left'].set_position(('outward', 10))
                ax.spines['bottom'].set_position(('outward', 10))
                # Hide the right and top spines
                ax.spines['right'].set_visible(False)
                ax.spines['top'].set_visible(False)
                # Only show ticks on the left and bottom spines
                ax.yaxis.set_ticks_position('left')
                ax.xaxis.set_ticks_position('bottom')

                ax.set_xticks(range(len(ticks)))
                ax.set_yticks(range(len(ticks)))

                ax.set_xticklabels(ticks)
                ax.set_yticklabels(ticks)

                ax.set_title(axtit)

                if c % 4 == 0:
                    ax.set_axis_off()

                c += 1

    fig.show()

##==============================================================================
def tanimoto_distances_matrix (param_grid, temporal_filtering, global_nuis_correction, prefs_method, prefs_thr):

    from scipy.spatial.distance import rogerstanimoto

    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    #loop will load all classification results dataf (or classify again) and calculate performance metrics
    param_gridlist = list(ParameterGrid(param_grid))
    n = len(param_gridlist)

    tanis = np.zeros((n,n))
    ticks = []

    for j1idx, j1 in enumerate(param_gridlist):

        subjsf    = j1['subjsf']
        cl        = j1['cl']

        fidx   = dataf.index(subjsf)
        maskf  = masks[fidx]

        result_file1 = os.path.join(wd, get_resultfile_name (subjsf, cl, prefs_method, prefs_thr))
        presels_vol1 = get_localizations_from_datashelf(result_file1, subjsf, labelsf, maskf)

        ticks.append(subjsf.split('_')[1])

        print('Tanimoto comparing ' + result_file1)

        for j2idx, j2 in enumerate(param_gridlist):

            subjsf    = j2['subjsf']
            cl        = j2['cl']

            fidx   = dataf.index(subjsf)
            maskf  = masks[fidx]

            result_file2 = os.path.join(wd, get_resultfile_name (subjsf, cl, prefs_method, prefs_thr))
            presels_vol2 = get_localizations_from_datashelf(result_file2, subjsf, labelsf, maskf)

            print('with ' + result_file2)

            tanis[j1idx, j2idx] = rogerstanimoto(presels_vol1.flatten().astype(bool), presels_vol2.flatten().astype(bool))

    return tanis, ticks

#http://matplotlib.org/examples/ticks_and_spines/spines_demo_dropped.html
##==============================================================================
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_cobre_classification as cs
from sklearn.grid_search import ParameterGrid

overwrite=True

aq_thresh = 0.99

#atlasquery atlases
atlases = []
atlases.append('MNI Structural Atlas')
atlases.append('Harvard-Oxford Cortical Structural Atlas')
atlases.append('Harvard-Oxford Subcortical Structural Atlas')

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False], 'atlas': atlases}

for j in list(ParameterGrid(preprocess_grid)):
    cs.do_autoaq(j['temporal_filtering'], j['global_nuis_correction'], overwrite, j['atlas'], aq_thresh)
'''
def do_autoaq (temporal_filtering=True, global_nuis_correction=True, overwrite=False, atlas='MNI Structural Atlas', aq_thresh=0.95):

    from sklearn.grid_search import ParameterGrid

    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    au.setup_logger(2, logfname=None)

    stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()
    #param_grid    = {'subjsf': dataf, 'cl': clf_methods, 'prefs' : prefs_methods, 'prefs_thr': prefs_thrs, 'fs1' : ['none']}

    res = [k for k in dataf if 'reho' in k]
    param_grid    = {'subjsf': res, 'cl': ['linsvm'], 'prefs' : ['pearson', 'bhattacharyya'], 'prefs_thr': [90, 95], 'fs1' : ['none']}

    #cluster threshold
    clthresh = 0.95

    for j in list(ParameterGrid(param_grid)):

        subjsf    = j['subjsf']
        cl        = j['cl']
        prefs     = j['prefs']
        prefs_thr = j['prefs_thr']
        #fs1 = j['fs1']
        #fs2 = j['fs2']

        #of = os.path.join(wd, au.remove_ext(subjsf))
        of = os.path.join('/home/alexandre/Data/cobre/autoaq', au.remove_ext(os.path.basename(subjsf)))
        if prefs != 'none': of += '_' + prefs + '_' + str(prefs_thr)
        #if fs1   != 'none': of += '_' + fs1
        #if fs2   != 'none': of += '_' + fs2
        #of += '_' + cl

        #checking if output files exist
        out_file_name = of
        if atlas == 'MNI Structural Atlas':
            out_file_name += '_mni_struct'
        elif atlas == 'Harvard-Oxford Cortical Structural Atlas':
            out_file_name += '_harvox_cort'
        elif atlas == 'Harvard-Oxford Subcortical Structural Atlas':
            out_file_name += '_harvox_subcort'

        out_autoaq_txt = out_file_name + '_autoaq_presels.txt' #presels is the same for all classifiers

        of = of + '_' + cl
        svsqryf = of + '_autoaq_sup_vecs.txt'

        if not overwrite:
            if os.path.exists(out_autoaq_txt) and os.path.exists(svsqryf):
                au.log.info ("File " + out_autoaq_txt + " already exists, jumping to next.")
                continue

        au.log.info("Running " + prefs + " " + str(prefs_thr) + " " + str(j))

        #loading data file
        data_file = os.path.join(wd, 'features', os.path.basename(of) + '.pyshelf')
        au.log.info ('Loading ' + data_file)
        res = shelve.open(data_file)

        try:
            presels = res['presels']
            cv      = res['cv']
            sv      = res['importance']
        except:
            print('Problems reading ' + data_file)
            res.close()

        #Loading data within cv in order to unmask feature_importances_

        fidx   = dataf.index(subjsf)
        maskf  = masks[fidx]
        tmpltf = templates[fidx]

        mask, hdr, aff = au.get_nii_data(maskf)
        #maskidx = np.array(np.where(mask > 0))
        #hdr.set_data_dtype(np.dtype(np.float))

        tmplt  = nib.load(tmpltf).get_data()

        X = np.load(subjsf)
        y = np.loadtxt(labelsf)

        #getting localizations from data
        presels_vol, supvecs_vol, supvecs_done = get_localizations (X, y, cv, maskf, presels, sv)

        #standardizing presels
        from sklearn.preprocessing import MinMaxScaler
        mms = MinMaxScaler()
        presels_vol = mms.fit_transform(presels_vol)

        #binarizing presels
        #from sklearn.preprocessing import Binarizer
        #binr = Binarizer()
        #presels_vol = binr.fit_transform(presels_vol)

        print(out_autoaq_txt)

        #presels autoaq
        preselsf = os.path.join(wd, 'presels.nii.gz')
        au.save_nibabel (preselsf, presels_vol, aff, hdr)

        #resample to 2mm ?
        if maskf.find('3mm') > -1:
            au.log.info ('Resampling presels.')
            presels2f = os.path.join(wd, 'presels2mm.nii.gz')
            au.exec_command('3dresample -dxyz 2.0 2.0 2.0 -prefix ' + presels2f + ' -inset ' + preselsf)
            os.remove(preselsf)
            os.rename(presels2f, preselsf)

        #calling autoaq
        #http://brainder.org/tag/autoaq/
        prqryf = out_autoaq_txt
        call_autoaq (xd, preselsf, aq_thresh, atlas, prqryf)

        os.remove(preselsf)

        #sup_vecs/importances autoaq
        if supvecs_done:
            supvecsf = os.path.join(wd, 'sup_vecs.nii.gz')
            au.save_nibabel (supvecsf, supvecs_vol, aff, hdr)

            #resample to 2mm ?
            if maskf.find('3mm') > -1:
                au.log.info ('Resampling sup_vecs.')
                supvecs2f = os.path.join(wd, 'sup_vecs2mm.nii.gz')
                au.exec_command('3dresample -dxyz 2.0 2.0 2.0 -prefix ' + supvecs2f + ' -inset ' + supvecsf)
                os.remove(supvecsf)
                os.rename(supvecs2f, supvecsf)

            #svsqryf = of + '_autoaq_sup_vecs.txt'
            call_autoaq (xd, supvecsf, aq_thresh, atlas, svsqryf)

            os.remove(supvecsf)

##==============================================================================
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_cobre_classification as cs
from sklearn.grid_search import ParameterGrid

overwrite=True

#atlasquery atlases
atlases = []
#atlases.append('MNI Structural Atlas')
atlases.append('Harvard-Oxford Cortical Structural Atlas')
atlases.append('Harvard-Oxford Subcortical Structural Atlas')

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False], 'atlas': atlases}

for j in list(ParameterGrid(preprocess_grid)):
    cs.do_atlasquery(j['temporal_filtering'], j['global_nuis_correction'], overwrite, j['atlas'])
'''
def do_atlasquery (temporal_filtering=True, global_nuis_correction=True, overwrite=False, atlas='MNI Structural Atlas'):

    from sklearn.grid_search import ParameterGrid

    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    au.setup_logger(0, logfname=None)

    stratified, stddize, cvfolds, thrmethod, clf_methods, prefs_methods, prefs_thrs = get_experiment_parameters()
    #param_grid    = {'subjsf': dataf, 'cl': clf_methods, 'prefs' : prefs_methods, 'prefs_thr': prefs_thrs, 'fs1' : ['none']}

    res = [k for k in dataf if 'reho' in k]
    param_grid    = {'subjsf': res, 'cl': ['linsvm'], 'prefs' : ['pearson', 'bhattacharyya'], 'prefs_thr': [90, 95], 'fs1' : ['none']}

    #cluster threshold
    clthresh = 0.95

    for j in list(ParameterGrid(param_grid)):

        subjsf    = j['subjsf']
        cl        = j['cl']
        prefs     = j['prefs']
        prefs_thr = j['prefs_thr']
        #fs1 = j['fs1']
        #fs2 = j['fs2']

        #of = os.path.join(wd, au.remove_ext(subjsf))
        of = os.path.join('/home/alexandre/Data/cobre/autoaq', au.remove_ext(os.path.basename(subjsf)))
        if prefs != 'none': of += '_' + prefs + '_' + str(prefs_thr)
        #if fs1   != 'none': of += '_' + fs1
        #if fs2   != 'none': of += '_' + fs2
        #of += '_' + cl

        #checking if output files exist
        out_file_name = of
        if atlas == 'MNI Structural Atlas':
            out_file_name += '_mni_struct'
        elif atlas == 'Harvard-Oxford Cortical Structural Atlas':
            out_file_name += '_harvox_cort'
        elif atlas == 'Harvard-Oxford Subcortical Structural Atlas':
            out_file_name += '_harvox_subcort'

        out_atlasquerpy_txt = out_file_name + '_atlasquerpy.txt'
        out_atlasquery_txt = out_file_name + '_atlasquery.txt'

        if not overwrite:
            if os.path.exists(out_atlasquery_txt):
                au.log.info ("File " + out_atlasquery_txt + " already exists, jumping to next.")
                continue

        au.log.info("Running " + prefs + " " + str(prefs_thr) + " " + str(j))

        #loading data file
        data_file = os.path.join(wd, 'features', os.path.basename(of) + '.pyshelf')
        if not os.path.exists(data_file):
            oldf = data_file.replace('.pyshelf', '_linsvm.pyshelf')
            newf = data_file
            os.rename(oldf, newf)
            #au.log.error('File ' + data_file + ' does not exist. Skipping to the next.')
            #continue

        au.log.info ('Loading ' + data_file)
        res = shelve.open(data_file)

        try:
            presels = res['presels']
            cv      = res['cv']
            sv      = res['importance']
        except:
            print('Problems reading ' + data_file)
            res.close()

        #Loading data within cv in order to unmask feature_importances_

        fidx   = dataf.index(subjsf)
        maskf  = masks[fidx]
        tmpltf = templates[fidx]

        mask, hdr, aff = au.get_nii_data(maskf)
        #maskidx = np.array(np.where(mask > 0))
        #hdr.set_data_dtype(np.dtype(np.float))

        tmplt  = nib.load(tmpltf).get_data()

        X = np.load(subjsf)
        y = np.loadtxt(labelsf)

        #getting localizations from data
        presels_vol, supvecs_vol, supvecs_done = get_localizations (X, y, cv, maskf, presels, sv)

        #standardizing presels
        from sklearn.preprocessing import MinMaxScaler
        mms = MinMaxScaler()
        presels_vol = mms.fit_transform(presels_vol)

        #binarizing presels
        #from sklearn.preprocessing import Binarizer
        #binr = Binarizer()
        #presels_vol = binr.fit_transform(presels_vol)

        #saving presels nifti file
        preselsf = os.path.join(wd, 'presels.nii.gz')
        au.save_nibabel (preselsf, presels_vol, aff, hdr)

        #resample to 2mm ?
        if maskf.find('3mm') > -1:
            au.log.info ('Resampling presels.')
            presels2f = os.path.join(wd, 'presels2mm.nii.gz')
            au.exec_command('3dresample -dxyz 2.0 2.0 2.0 -prefix ' + presels2f + ' -inset ' + preselsf)
            os.remove(preselsf)
            os.rename(presels2f, preselsf)

        #binarise
        au.exec_command('fslmaths ' + preselsf + ' -bin ' + preselsf)

        #calling atlasquery
        #saving binary presels file first
        call_atlasquerpy(xd, preselsf, atlas, out_atlasquerpy_txt)
        #call_atlasquery(xd, preselsf, atlas, out_atlasquery_txt)

        os.remove(preselsf)

##==============================================================================
##Transforming numpy data format to matlab format
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_classification as cs
from sklearn.grid_search import ParameterGrid

insert_aal_idx = True

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

for j in list(ParameterGrid(preprocess_grid)):
    cs.do_npy_to_mat(**j, insert_aal_idx)
'''
def do_npy_to_mat (temporal_filtering=True, global_nuis_correction=True, insert_aal_idx=False):
    import nibabel as nib
    import scipy.io as sio

    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    au.setup_logger(2, logfname=None)

    for fi in range(len(dataf)):

        subjsf = dataf[fi]
        maskf = masks[dataf.index(subjsf)]

        X = np.load(subjsf)
        y = np.loadtxt(labelsf)
        mask = nib.load(maskf).get_data()
        indices = np.where(mask > 0)

        #for COBRE ONLY, removing last two subjects:
        X = X[:-2,:]
        y = y[:-2]

        data = {'X': X.transpose(), 'y': y, 'mask': mask, 'mask_indices': indices}

        #should I insert AAL indices?
        if insert_aal_idx:
            if maskf.find('3mm') > -1:
                aalf = os.path.join(xd, 'aal_3mm.nii.gz')
            else:
                aalf = os.path.join(xd, 'aal_2mm.nii.gz')

            aal = nib.load(aalf).get_data()
            aal_indices = aal[indices]
            data.update({'aal_indices': aal_indices})

        of = au.remove_ext(subjsf) + '.mat'

        print('Saving ' + of)

        sio.savemat(of, data)

##==============================================================================
##Transforming numpy data format to matlab format, with prefeatsel
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_classification as cs

from sklearn.grid_search import ParameterGrid

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

for j in list(ParameterGrid(preprocess_grid)):
    cs.do_npy_to_mat(**j)
'''
def do_npy_to_mat_with_prefeats():
    import nibabel as nib
    import scipy.io as sio

    temporal_filtering     = True
    global_nuis_correction = False
    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    verbose   = 2
    au.setup_logger(verbose, logfname=None)

    #pre feature selection processes
    prefs_methods = ['pearson', 'bhattacharyya', 'welcht']
    prefs_thrs = [80, 85, 90, 95, 99, 99.5]

    for fi in range(len(dataf)):

        subjsf = dataf[fi]
        maskf = masks[dataf.index(subjsf)]

        X = np.load(subjsf)
        y = np.loadtxt(labelsf)
        mask = nib.load(maskf).get_data()
        indices = np.where(mask > 0)

        #for COBRE ONLY, removing last two subjects:
        X = X[:-2,:]
        y = y[:-2]

        pearson = aizc.pre_featsel (X, y, 'pearson', 95)

        X_filt  = X[:, pearson > 0]

        data = {'X': X, 'y': y, 'mask': mask, 'mask_indices': indices, 'X_filt': X_filt, 'pearson': pearson}

        of = au.remove_ext(subjsf) + '.mat'

        sio.savemat(of, data)

#===============================================================================
# Creating localization images for Darya
def do_darya_localizations ():
    import scipy.io as sio

    sys.path.append('/home/alexandre/Dropbox/Documents/work/visualize_volume')
    import visualize_volume as vis

    import aizkolari_utils as au
    au.setup_logger()

    locd = '/home/alexandre/Dropbox/ELM 2013/ELM-2013-Darya/localization'

    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths()

    maskf          = dilmasks[0]
    mask, hdr, aff = au.get_nii_data(maskf)
    indices        = np.array(np.where(mask > 0))

    tmpltf         = templates[0]
    tmplt, _, _    = au.get_nii_data(tmpltf)

    flist = os.listdir(locd)
    flist = au.find(flist, '.mat')
    flist.sort()

    for f in flist:
        data = sio.loadmat(os.path.join(locd, f))

        name = au.remove_ext(f)

        if f.find('cobre') >= 0:
            p = data['pearson'].squeeze()
            locs = p > 0
            lidx = indices[:, locs].squeeze()

            if f.find('reho') >= 0:
                preho = p.copy()
            elif f.find('alff') >= 0:
                palff = p.copy()

        else:
            locs = data[name].squeeze()
            locs -= 1
            if f.find('pearson') >= 0:

                if f.find('reho') >= 0:
                    lidx = indices[:, preho > 0]
                elif f.find('alff') >= 0:
                    lidx = indices[:, palff > 0]

                lidx = lidx[:, locs]
            else:
                lidx = indices[:, locs].squeeze()

        locvol = np.zeros_like(mask, dtype=np.float)
        locvol[tuple(lidx)] = 1

        #save nifti volume
        au.save_nibabel (os.path.join(locd, name + '.nii.gz'), locvol, aff, hdr)

        #save image
        fig = vis.show_many_slices(tmplt, locvol, volaxis=1, vol2_colormap=plt.cm.autumn, figtitle=name)
        aizc.save_fig_to_png(fig, os.path.join(locd, name + '.png'))
        if os.access (os.path.join(xd,'innercrop'), os.X_OK):
            au.exec_command (os.path.join(xd,'innercrop') + ' -o white ' + fig2name + ' ' + fig2name)

#===============================================================================
##Transforming numpy data format to matlab format
'''
import sys
sys.path.append('/home/alexandre/Dropbox/Documents/work/cobre')
import do_classification as cs

from sklearn.grid_search import ParameterGrid

temporal_filtering     = [True, False]
global_nuis_correction = [True, False]
preprocess_grid = {'temporal_filtering': [True, False], 'global_nuis_correction': [True, False]}

for j in list(ParameterGrid(preprocess_grid)):
    cs.do_matlab_svm_bdopt(**j)
'''
def do_matlab_svm_bdopt (temporal_filtering=True, global_nuis_correction=True):
    wd, xd, dd, labelsf, phenof, dataf, masks, dilmasks, templates, pipe = get_filepaths(temporal_filtering, global_nuis_correction)

    au.setup_logger(2, logfname=None)

    for fi in range(len(dataf)):

        myfile = au.remove_ext(subjsf) + '.mat'


class Node:
	def __init__(self,data = 0):
		self.data = 0
		self.next = None
class LinkedList:
	def __init__(self):
		self.head = None
	def push(self,new_data):
		new_node = Node(new_data)
		if self.head == None:
			self.head = new_node
		else:
			temp = self.head
			while(temp.next is not None):
				temp = temp.next
			temp.next = new_node
			new_node.next = None

	def detectLoop(self):
		s = set()
		temp = self.head
		while temp:
			if temp in s:
				return True
			s.add(temp)
			temp = temp.next
		return False

llist = LinkedList()
llist.push(20)
llist.push(4)
llist.push(15)
llist.push(10)
llist.head.next.next.next.next = llist.head;
if llist.detectLoop():
	print("Loop Found")
else:
	print("No Loop")