# Author: Xiran Zhang, Pengfei Li
# training are implemented here

import os
import time
import argparse
import numpy as np
import cv2
from tqdm import tqdm
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter

import dataloader.sf_listfile as lf
import dataloader.sf_loader as ld
from model import LossFunc
from model import SPAutoED as SPEDNet
from utils.disp_utils import depth_to_disp

parser = argparse.ArgumentParser(description='SPAutoED')
parser.add_argument('--datapath', default='/media/xiran_zhang/Crypto/sceneflow/')
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--loadmodel', default=None, help='path to the trained model, for continuous training')
parser.add_argument('--savemodel', default='./trained_model/', help='folder to the save the trained model')
parser.add_argument('--seed', type=int, default=1, metavar='S', help='seed for random (default:1)')

args = parser.parse_args()
writer = SummaryWriter()

# set train environment
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
print('torch cuda status:', torch.cuda.is_available())
if torch.cuda.is_available():
    cuda = True
else:
    EnvironmentError('cuda status error.')

# load all data
all_left_img, all_right_img, test_left_img, test_right_img = lf.dataloader(args.datapath)

trainImgLoader = torch.utils.data.DataLoader(
         ld.myImageFloder(all_left_img,all_right_img, True),
         batch_size= 10, shuffle= True, num_workers= 5, drop_last=False)

valImgLoader = torch.utils.data.DataLoader(
         ld.myImageFloder(test_left_img,test_right_img, False),
         batch_size= 15, shuffle= False, num_workers= 5, drop_last=False)

# create model
model = SPEDNet.SPAutoED()
if cuda:
    model = nn.DataParallel(model)
    model = model.cuda()
pasued_epoch = -1

# set criteria, optimizer and scheduler
# Option can be made here:
# optimizer: Adam, SGD
criteria = LossFunc.Loss_reonstruct().cuda()
optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 5, 0.7)

# load model
if args.loadmodel is not None:
    state_dict = torch.load(args.loadmodel)
    model.load_state_dict(state_dict['state_dict'])
    pasued_epoch = state_dict['epoch']
    train_loss = state_dict['train_loss']
    # val_loss = state_dict['val_loss']
    scheduler = state_dict['scheduler']
    optimizer = state_dict['optimizer']
    # print('train loss:', train_loss, 'val loss:', val_loss)

print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))

# train function / validation function / test function
def train(imgL, imgR, model, optimizer):
    model.train()

    # plt.imshow(imgL[0,0])
    # plt.show()
    # plt.imshow(imgR[0,0])
    # plt.show()

    if cuda:
        imgL = imgL.cuda()
        imgR = imgR.cuda()

    optimizer.zero_grad()
    depthL = model(imgL)
    depthR = model(imgR)
    # depth to disparity
    # dispL, dispR = depth_to_disp(depthL, depthR, camera_para)

    dispL, dispR = depthL.squeeze(1), depthR.squeeze(1)

    # visualize
    # depthL_cpu = depthL.cpu()
    # depthR_cpu = depthR.cpu()
    # depthL_cpu.detach_()
    # depthR_cpu.detach_()
    # plt.imshow(depthL_cpu[0,0])
    # plt.show()
    # plt.imshow(depthR_cpu[0,0])
    # plt.show()

    left_ssim, right_ssim, left_l1, right_l1, lr_l1, rl_l1, smooth_left_disp, smooth_right_disp = criteria(imgL, imgR, dispL, dispR)

    start_plane = torch.ones_like(depthL)*10
    # reg = F.l1_loss(depthL, start_plane) + F.l1_loss(depthR,start_plane)

    print('left recons loss:', right_l1.data.item(), 'right recons loss:', left_l1.data.item(), \
        'left recons l1:', left_l1.data.item(), 'right recons l1:', right_l1.data.item(), \
        'lr consis loss:', lr_l1.data.item(), 'rl consis loss:', rl_l1.data.item(), \
         'smooth left disp:', smooth_left_disp, 'smooth right disp:', smooth_right_disp)

    loss = 0.85*(left_ssim + right_ssim) + 0.15*(left_l1 + right_l1) + 1.0 * (lr_l1 + rl_l1)/(960.0*0.2) + 0.1 * (smooth_left_disp + smooth_right_disp)/(960.0*0.2)
    # torch.autograd.set_detect_anomaly(True)
    loss.backward()
    optimizer.step()

    return loss.data.item()

def val(imgL, imgR, model, optimizer, epoch):
    model.eval()
    if cuda:
        imgL = imgL.cuda()
        imgR = imgR.cuda()

    with torch.no_grad():
        depthL = model(imgL)
        depthR = model(imgR)

        # visualization
        # fig, (ax1, ax2) = plt.subplots(1,2)
        # ax1.imshow(depthL)
        # ax2.imshow(dephtR)
        # plt.show()
        # dispL, dispR = depth_to_disp(depthL, depthR, camera_para)
        dispL, dispR = depthL.squeeze(1), depthR.squeeze(1)
        # reg = F.l1_loss(depthL) + F.l1_loss(depthR)
        left_ssim, right_ssim, left_l1, right_l1, lr_l1, rl_l1, smooth_left_disp, smooth_right_disp = criteria(imgL, imgR, dispL, dispR)
        loss = 0.85*(left_ssim + right_ssim) + 0.15*(left_l1 + right_l1) + 1.0 * (lr_l1 + rl_l1)/(960.0*0.2) + 0.1 * (smooth_left_disp + smooth_right_disp)/(960.0*0.2)

    return loss.data.item()

def main():
    start_epoch = pasued_epoch+1
    total_epochs = args.epochs
    print('Start training ...')

    total_iteration = 0
    for epoch in range(start_epoch, total_epochs):
        print('\nEPOCH ' + str(epoch + 1) + ' of ' + str(total_epochs) + '\n')
        # -----------------Training---------------
        train_batch_num = 0
        epoch_train_loss = 0

        for idx, (left, right) in enumerate(tqdm(trainImgLoader)):
            start_time = time.time()
            train_loss = train(left, right, model, optimizer)
            epoch_train_loss += train_loss
            train_batch_num += 1
            total_iteration += 1
            print('Iter %d training loss = %.3f, time =  %.2f' %(idx, train_loss, time.time() - start_time))
            writer.add_scalar('Iteration loss', train_loss, total_iteration+1)

        epoch_train_loss /= train_batch_num
        writer.add_scalar('Loss/train', epoch_train_loss, epoch+1)

        #----------------Validation--------------
        # val_batch_num = 0
        # epoch_val_loss = 0
        # for idx, (left, right) in enumerate(tqdm(valImgLoader)):
        #     start_time = time.time()
        #     val_loss = val(left, right, model, optimizer, epoch)
        #     epoch_val_loss += val_loss
        #     val_batch_num += 1
        #     print('Iter %d val loss = %.3f, time =  %.2f' %(idx, val_loss, time.time() - start_time))

        # epoch_val_loss /= val_batch_num
        # writer.add_scalar('Loss/val', epoch_val_loss, epoch+1)

        savefilename = args.savemodel + 'SPAutoED_' + str(epoch+1) + '.tar'
        torch.save({
            'epoch': epoch,
            'state_dict': model.state_dict(),
            'train_loss':epoch_train_loss,
            # 'val_loss':epoch_val_loss,
            'optimizer':optimizer,
            'scheduler':scheduler
        }, savefilename)

        scheduler.step()
        lr = scheduler.get_lr()

    writer.close()

if __name__ == '__main__':
    main()
    # test()


"""Support for a single sharing portal (was called published set) in a Skytap environment or vm."""
from skytap.models.SkytapResource import SkytapResource
from skytap.models.NewBase import NewSTResource
"""
Field Def schema
field_defs = dict(
    <field_name>=dict(
        type='<*str|date|datetime|time|timezone|int|bool|double|array>',
        required=True|*False,
        sub_rec_def=<schema dict | *None>,

* = defaults
"""
SHARING_PORTAL_WRITABLE_FIELDS = [
    'name', 'publish_set_type', 'runtime_limit', 'expiration_date', 'expiration_date_tz', 'start_time', 'end_time',
    'time_zone', 'password', 'sso_required', 'vms', 'vms.vm_ref', 'vms.access', 'anonymous_smart_rdp',
    'custom_content_enabled', 'custom_title', 'custom_content', 'custom_content_is_default']

SHARING_PORTAL_FIELD_DEFS = {
    'id': {'type': 'int'},
    'name': {'required': True},
    'runtime_limit': {'type': 'int'},
    'runtime_left_in_seconds': {'type': 'int'},
    'expiration_date': {'type': 'datetime', 'tz': 'expiration_date_tz'},
    'expiration_date_tz': {'type': 'timezone'},
    'start_time': {'type': 'time', 'tz': 'time_zone'},
    'end_time': {'type': 'time', 'tz': 'time_zone'},
    'time_zone': {'type': 'timezone'},
    'multiple_url': {'type': 'bool'},
    'sso_required': {'type': 'bool'},
    'vms.id': {'type': 'int'},
    'vms.use_and_run': {'type': 'bool'},
    'vms.error': {'type': 'bool'},
    'anonymous_smart_rdp': {'type': 'bool'},
    'custom_content_enabled': {'type': 'bool'},
    'custom_content_is_default': {'type': 'bool'},
}

class SharingPortal(NewSTResource, SkytapResource):

    """One note."""
    _writable_fields = SHARING_PORTAL_WRITABLE_FIELDS
    _field_defs = SHARING_PORTAL_FIELD_DEFS

    def __init__(self, portal_json):
        super(SharingPortal, self).__init__(portal_json)

        self._new_convert_data_elements()


class Solution(object):
    def permute(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        res = []
        def _permute(nums, curr, k):
            if len(curr) == k:
                res.append(curr[:])
                return

            for i in nums:
                if i not in curr:
                    curr.append(i)
                    _permute(nums, curr, k)
                    curr.pop()

        _permute(nums, [], len(nums))
        return res

    def permute(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        res = []

        def _permute(nums, curr):
            if nums == []:
                res.append(curr)
                return

            for i in range(len(nums)):
                _permute(nums[:i] + nums[i+1:], curr + [nums[i]])

        _permute(nums, [])
        return res


import json
import numpy as np
from scipy.spatial.distance import euclidean as distance_

dimensions_ingredients = 702
dimensions_nutrition_facts = 22
number_of_clusters=100

filename = "recipes.json"
with open(filename) as json_data:
    recipes = json.load(json_data)
    names = list(recipes.keys())
    X = np.zeros([len(names), dimensions_ingredients+dimensions_nutrition_facts])
    labels = {}
    index = 0
    for label, recipe in recipes.items():
        for ingredient in recipe["ingredients"]:
            X[index, ingredient[0]] = ingredient[1]
        for fact in recipe["nutrition"]:
            X[ index, dimensions_ingredients + fact[0] ] = fact[1]

        labels[index] = recipe["healthLabels"]
        index+=1
print("File "+filename+" was successfully read")
#names = [label for label, recipe in recipes.items()]

class IdssFood:
    # Carlos: Define the clusters, just a list
    CLUSTERS = {
        "starters": [],
        "mains": [],
        "desserts": []
    }
    NAMES = names
    RECIPES = recipes
    LABELS = labels
    X_ = X

    def __init__(self):
        self.liked = []
        self.disliked = []

        self.labels = None  # AND
        self.no_labels = []  # AND

        self.ingredients = None  # OR
        self.no_ingredients = []  # AND

    def set_liked(self, liked=[], disliked=[]):
        self.liked = self.label2index(self,liked)
        self.disliked = self.label2index(self,disliked)

    def set_labels(self, labels=None, no_labels=()):
        self.labels = labels
        self.no_labels = no_labels

    def set_ingredients(self, ingredients=None, no_ingredients=()):
        self.ingredients = ingredients
        self.no_ingredients = no_ingredients

    def get_closest_to_liked(self, cluster=None):
        # Marti: Use k-nearest neighbours having into account the valid clusters
        closest =list(range(20))
        return closest

    def filter(self, closest):
        # Nariman: Use labels, no_labels and ingredients and no_ingredients to filter out
        remove_list = []
        for index in closest:
            #labels filtering
            for label in  self.LABELS[index]:
                if label in self.no_labels:
                    remove_list += [index]
            for label in  self.labels:
                if label not in self.LABELS[index]:
                    remove_list += [index]
            #ingredients filtering            ###this is based on having X but it can easily changed to get from json file directly if needed.
            for ingredient in  self.no_ingredients:
                if self.X_[index,ingredient]:
                    remove_list += [index]
            for ingredient in  self.ingredients:
                if self.X_[index,ingredient]==0:
                    remove_list += [index]
        for item in set(remove_list):
            closest.remove(item)
        for item in self.disliked:
            if item in closest:
                closest.remove(item)
        return closest

    def rank(self, closest,utility = 0): # utility=0:OWA, utility=1:meanstd
        # Nariman: Use disliked dishes to rank
        #first approach: utility function => mean-std
        L = len(closest)
        X_ingredients = self.X_[:,0:dimensions_ingredients]

        if utility==1:
            dist_list = []
            for i in range(L):
                temp = []
                for dish in self.disliked:
                    temp.append( distance_(X_ingredients[int(dish),:],X_ingredients[int(closest[i]),:]) )
                dist_list.append(temp)
            rank_list = []
            for index in range(len(dist_list)):
                rank = 1*np.mean(dist_list[index])-1*np.std(dist_list[index])
                rank_list.append([rank, index])
            rank_list.sort(reverse=True)
            closest_temp = []
            for i in range(0,len(closest)):
                closest_temp.append(closest[int(rank_list[i][1])])
            closest = closest_temp
            return closest
        #second approach: utility function => OWA :  more weight for closer distances to dislike
        dist_list = []
        for i in range(L):
            temp = []
            for dish in self.disliked:
                temp.append( distance_(X_ingredients[int(dish),:],X_ingredients[int(closest[i]),:]) )
            dist_list.append([temp,i])
        L = len(closest)
        OWAweights = []
        if L != 1:
            for w in range(len(self.disliked)):
                OWAweights.append(2*(w+1)/((len(self.disliked))*((len(self.disliked))+1)))
        else: OWAweights = [1]
        rank_list_ = []
        dist_list.sort(reverse=True)
        for index in range(L):
            rank_ = sum(np.multiply(dist_list[index][0], OWAweights))
            rank_list_.append([rank_,dist_list[index][1]])
        rank_list_.sort(reverse=True)
        closest_temp_ = []
        for i in range(L):
            closest_temp_.append(closest[int(rank_list_[i][1])])
        closest = closest_temp_
        return closest

    def index2label(self, list):
        return [IdssFood.NAMES[index] for index in list]

    def label2index(self, list):
        return [IdssFood.NAMES.index(label) for label in list]

    def get_recommendation(self, clusters=("starters", "mains", "desserts")):
        recommendations = {"starters": None, "mains": None, "desserts": None}
        for cluster in clusters:
            recipes = self.get_closest_to_liked(cluster)
            recipes = self.filter(recipes)
            recipes = self.rank(recipes)
            recommendations[cluster] = self.index2label(recipes)
        return recommendations

TEST = IdssFood
TEST.set_labels(TEST, labels=['Gluten-Free'], no_labels=[])
TEST.set_ingredients(TEST, ingredients=[], no_ingredients=[])
TEST.set_liked(TEST, disliked=['Persimmon Tart','Cumin Lime Black Bean Quinoa Salad'])
x=TEST.get_closest_to_liked(TEST)
x=TEST.filter(TEST,x)
print(x)
x=TEST.rank(TEST,x)
y=TEST.index2label(TEST, x)
print(x,'\n',y)


"""
Chuhang Zou
07.2019

Code Revised from:

Finetuning Torchvision Models
=============================

**Author:** `Nathan Inkawhich <https://github.com/inkawhich>`__

"""

from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torchvision import models, transforms
import copy
print("PyTorch Version: ",torch.__version__)

import time
from torch.utils import data
import scipy.io as sio
from model_occ import *
from data_generator_occ import *
import re

# Top level data directory. Here we assume the format of the directory conforms
#   to the ImageFolder structure
data_dir = "./data/ShapeNet/"
model_path = "./model/PointCloud_Reconstruction_resnet50_Complete_Silhouette_Guidance.pth"

# use train-val since we evaluate on Pix3D but not ShapeNet
train_datapath = './data/trainval_list.txt'
val_datapath = './data/test_list.txt'

# Pre-trained models to choose from [resnet18, resnet34, resnet50]
model_name = "resnet50"

# load pretrained weights
Flag_loadweights = True
weight_path = './model/PointCloud_Reconstruction_resnet50_ShapeNet.pth'

# Number of classes in the dataset
num_classes = 1024

# grid size for refinement branch
grid_size = 2

# Batch size for training (change depending on how much memory you have)
batch_size = 32

# Number of epochs to train for
num_epochs = 100000
steps_per_epoch = 20

# Model Training and Validation Code
def train_model(model, train_generator, val_generator, optimizer, steps=100, num_epochs=25):
    since = time.time()

    val_acc_history = []

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = np.Inf

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
                dataloaders = train_generator
            else:
                model.eval()   # Set model to evaluate mode
                dataloaders = val_generator

            loss_3d_f_sum = 0.0
            loss_2d_f_sum = 0.0
            loss_2d_f_t_sum = 0.0
            loss_sum = 0.0
            step = 0

            # Iterate over data.
            for input in dataloaders:

                inputs = input[0]
                labels = input[1]
                masks = input[2]

                # gpu mode
                inputs = inputs.to(device)
                labels_f = labels.to(device)
                masks = masks.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    # Get model outputs and calculate loss

                    # silhouette
                    masks = (masks>0.5).float()
                    inputs = torch.cat((inputs, masks),1)

                    _, outputs_f = model(inputs)

                    # loss
                    dist11, dist22 = distChamfer_bn(outputs_f, labels_f)

                    loss_2d_f = distChamfer_2d(outputs_f, masks)
                    loss_2d_f_t = distChamfer_2d_t(outputs_f, labels_f)

                    w1 = torch.mean(dist11)
                    w2 = torch.mean(dist22)
                    w1 = w1.data
                    w2 = w2.data

                    loss_3d_f = (torch.mean(dist11))+(torch.mean(dist22))
                    loss_2d_f = loss_2d_f*0.000000001
                    loss_2d_f_t = loss_2d_f_t*0.0000000001

                    loss = loss_3d_f + loss_2d_f + loss_2d_f_t

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
                        optimizer.step()

                # statistics
                loss_3d_f_sum += loss_3d_f.item()*inputs.size(0)/steps
                loss_2d_f_sum += loss_2d_f.item()*inputs.size(0)/steps
                loss_2d_f_t_sum += loss_2d_f_t.item()*inputs.size(0)/steps
                loss_sum += (w1+w2)*inputs.size(0)/steps

                # clear cache
                #torch.cuda.empty_cache()

                # Break after 'steps' steps
                if step==steps-1:
                    break
                step += 1

            print('{} Loss: {:.6f}, Loss 3d: {:.6f}, Loss 2d: {:.6f},  Loss 2d t: {:.6f}'.format(phase, loss_sum, loss_3d_f_sum, loss_2d_f_sum*10000, loss_2d_f_t_sum*10000))

            # deep copy the model
            if phase == 'val' and best_acc > loss_sum:
                best_acc = loss_sum
                best_model_wts = copy.deepcopy(model.state_dict())
                # save model
                torch.save({'model_state_dict':best_model_wts, 'best_acc':best_acc}, model_path)
                print("Model saved ...")
            if phase == 'val':
                val_acc_history.append(loss_sum)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:6f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history

print("Load Models...")
# Define the encoder
encoder = initialize_encoder(model_name, num_classes,use_pretrained=True)

# Full model
model_ft = OccNet(encoder, num_classes, grid_size)

# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Send the model to GPU
model_ft = model_ft.to(device)

# Model initialization
set_parameter_requires_grad(model_ft)
# Print the model we just instantiated
#print(model_ft)
# if load weights
if Flag_loadweights:
    # pretrained header
    pretrained_dict = torch.load(weight_path)
    model_dict = model_ft.state_dict()
    pretrained_dict['resnet.conv1.weight'] = torch.cat((pretrained_dict['resnet.conv1.weight'], model_dict['resnet.conv1.weight'][:,3:,:,:]), 1)
    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
    model_dict.update(pretrained_dict)
    model_ft.load_state_dict(model_dict)

# update header
print("Params to learn:")
for name,param in model_ft.named_parameters():
#    # tune hearder only
#    if name != 'conv1.bias' and name != 'conv1.weight' and name != 'conv2.weight' and name != 'conv2.bias' and name != 'conv3.bias' and name != 'conv3.weight':
#        param.requires_grad = False

    if param.requires_grad == True:
        print("\t",name)

# Gather the parameters to be optimized/updated in this run.
params_to_update = [param for name, param in model_ft.named_parameters() if param.requires_grad]
#params_to_update = model_ft.parameters()
#time.sleep(500)

# Create the Optimizer
optimizer_ft = optim.Adam(params_to_update, lr = 1e-4, eps = 1e-6)

# Load Data
print("Initializing Datasets and Dataloaders...")
train_set = ShapeNetDataset(train_datapath, data_dir, 'train', grid_size=grid_size, transform=True)
train_generator = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=4)
val_set = ShapeNetDataset(val_datapath, data_dir, 'val', grid_size=grid_size, transform=True)
val_generator = torch.utils.data.DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=4)

# Train and evaluate
model_ft, hist = train_model(model_ft, train_generator, val_generator, optimizer_ft, steps_per_epoch, num_epochs=num_epochs)

print('training done')


from __future__ import division
from math import hypot

class RightEye(object):

    RIGHT_EYE_POINTS = [42, 43, 44, 45, 46, 47]

    def __init__(self, frame, landmarks, index):
        self.check = None
        self.index = None
        self.decision = None
        self.analyze(frame, landmarks, index)

    @staticmethod
    def midpoint(p1, p2):
        x = int((p1.x + p2.x) / 2)
        y = int((p1.y + p2.y) / 2)
        return x, y

    def analyze(self, frame, landmarks, index):
        points = self.RIGHT_EYE_POINTS
        self.right_blink(landmarks, points)
        self.index = index

    def right_blink(self, landmarks, points):
        left_point = (landmarks.part(points[0]).x, landmarks.part(points[0]).y)
        right_point = (landmarks.part(points[3]).x, landmarks.part(points[3]).y)
        center_top = self.midpoint(landmarks.part(points[1]), landmarks.part(points[2]))
        center_bottom = self.midpoint(landmarks.part(points[5]), landmarks.part(points[4]))

        hor_line_length = hypot((left_point[0] - right_point[0]), (left_point[1] - right_point[1]))
        ver_line_length = hypot((center_top[0] - center_bottom[0]), (center_top[1] - center_bottom[1]))

        ratio = (hor_line_length - ver_line_length) / (2*ver_line_length)
        self.check = ratio

    def control(self):
        if self.check > 2:
            self.decision = 1
            return self.decision
        else:
            self.decision = 0
            return self.decision

    def decide(self):
        if self.index == 1:
            return self.decision
        else:
            self.decision = 0
            return self.decision


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 27 11:23:51 2020

@author: francois

PJ5 Openclassrooms : this script is prerequisites for Grid search validation and saves results
To be used as prerequisite of PJ5_GridSearch1.py

"""

#%matplotlib inline

#%load_ext autoreload  # Autoreload has a bug : when you modify function in source code and run again, python kernel hangs :(
#%autoreload 2

import datetime as dt

import sys, importlib

from functions import *
#importlib.reload(sys.modules['functions'])

import pandas as pd

pd.set_option('display.max_rows', 200)
pd.set_option('display.max_columns', 200)

import datetime as dt

import os
import zipfile
import urllib

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

import qgrid

import glob

from pandas.plotting import scatter_matrix

from sklearn.model_selection import StratifiedShuffleSplit

from sklearn.model_selection import GridSearchCV

from sklearn.manifold import TSNE
from sklearn.manifold import LocallyLinearEmbedding
from sklearn.manifold import Isomap

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.metrics import pairwise_distances
from sklearn.cluster import AgglomerativeClustering
from scipy.stats import entropy

from sklearn.feature_selection import RFE

DATA_PATH = os.path.join("datasets", "onlineretail")
DATA_PATH = os.path.join(DATA_PATH, "out")

DATA_PATH_FILE_INPUT = os.path.join(DATA_PATH, "OnlineRetail_transformed.csv")

ALL_FEATURES = []

#MODEL_FEATURES=['InvoiceNo', 'InvoiceDate', 'CustomerID', 'TotalPrice', 'DescriptionNormalized', 'InvoiceMonth', 'StockCode']
MODEL_CLIENT_FEATURES = ['TotalPricePerMonth', 'DescriptionNormalized', 'HasEverCancelled', 'BoughtTopValueProduct' ]

plt.rcParams["figure.figsize"] = [16,9] # Taille par défaut des figures de matplotlib

import seaborn as sns
sns.set()

#import common_functions

####### Paramètres pour sauver et restaurer les modèles :
import pickle
####### Paramètres à changer par l'utilisateur selon son besoin :

'''
RECOMPUTE_GRIDSEARCH = True  # CAUTION : computation is several hours long
SAVE_GRID_RESULTS = False # If True : grid results object will be saved to pickle files that have GRIDSEARCH_FILE_PREFIX
LOAD_GRID_RESULTS = False # If True : grid results object will be loaded from pickle files that have GRIDSEARCH_FILE_PREFIX
                          # Grid search results are loaded with full samples (SAMPLED_DATA must be False)
'''

RECOMPUTE_GRIDSEARCH = True  # CAUTION : computation is several hours long
SAVE_GRID_RESULTS = True # If True : grid results object will be saved to pickle files that have GRIDSEARCH_FILE_PREFIX
LOAD_GRID_RESULTS = False # If True : grid results object will be loaded from pickle files that have GRIDSEARCH_FILE_PREFIX

#GRIDSEARCH_CSV_FILE = 'grid_search_results.csv'

GRIDSEARCH_FILE_PREFIX = 'grid_search_results_'

EXECUTE_INTERMEDIATE_MODELS = True # If True: every intermediate model (which results are manually analyzed in the notebook) will be executed

# Necessary for predictors used in the notebook :
from sklearn.linear_model import LinearRegression
from sklearn.compose import TransformedTargetRegressor
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error

from sklearn.preprocessing import PolynomialFeatures

### For progress bar :
from tqdm import tqdm_notebook as tqdm

# Statsmodel :
import statsmodels.formula.api as smf

import statsmodels.api as sm
from scipy import stats

SAVE_API_MODEL = True # If True : API model ill be saved
API_MODEL_PICKLE_FILE = 'API_model_PJ5.pickle'

# Load data
print('Load data')

df = load_data(DATA_PATH_FILE_INPUT)

df, df_train, df_test = custom_train_test_split_sample(df, 'TotalPrice')
df_train.reset_index(inplace=True)
df_test.reset_index(inplace=True)
df_train_ori = df_train.copy(deep=True)
df_test_ori = df_test.copy(deep=True)

# Top value products (must be saved with the model, and passed to it)
print('Top value products (must be saved with the model, and passed to it)')
df_nocancel = df_train[df_train['InvoiceNo'].str.startswith('C') == False]
df_nocancel.reset_index(inplace=True)

df_gbproduct = df_nocancel[['StockCode', 'TotalPrice']].groupby('StockCode').sum()['TotalPrice']

TOP_VALUE_PRODUCT_THRESHOLD = 20
top_value_products = df_gbproduct.sort_values(ascending=False).head(TOP_VALUE_PRODUCT_THRESHOLD).index  # Get top value products

# Preparation pipeline : model with bow features + TotalPricePerMonth + BoughtTopValueProduct + HasEverCancelled
print('Preparation pipeline : model with bow features + TotalPricePerMonth + BoughtTopValueProduct + HasEverCancelled')

preparation_pipeline = Pipeline([
    #('features_selector', FeaturesSelector(features_toselect=MODEL_FEATURES)),
    ('bow_encoder', BowEncoder()),
    ('agregate_to_client_level', AgregateToClientLevel(top_value_products)),
    ('scaler', LogScalerMultiple(features_toscale=['TotalPricePerMonth'])),

    # Faire la réduction dimensionnelle à part pour les bag of words et pour les autres features

    ('minmaxscaler', MinMaxScalerMultiple(features_toscale=['TotalPricePerMonth'])),
    ('dimensionality_reductor', DimensionalityReductor(features_totransform=['DescriptionNormalized'], \
                                                        algorithm_to_use='PCA', n_dim=200)),
    ('minmaxscaler_final', MinMaxScalerMultiple(features_toscale='ALL_FEATURES')),
])

df_train = preparation_pipeline.fit_transform(df_train)
df_test = preparation_pipeline.transform(df_test)

series_total_price_per_month_train = df_train['TotalPricePerMonth']
series_total_price_per_month_test = df_test['TotalPricePerMonth']

series_hasevercancelled_train = df_train['HasEverCancelled']
series_hasevercancelled_test = df_test['HasEverCancelled']

series_boughttopvalueproduct_train = df_train['BoughtTopValueProduct']
series_boughttopvalueproduct_test = df_test['BoughtTopValueProduct']

# Generation of TotalPricePerMonth labels
pca = PCA(n_components=2, random_state=42)
X_transformed = pca.fit_transform(df_train)
X_test_transformed = pca.fit_transform(df_test)

print('Binarisation of color categories')
bins = [-np.inf,df_train['TotalPricePerMonth'].quantile(0.25),\
        df_train['TotalPricePerMonth'].quantile(0.50),\
        df_train['TotalPricePerMonth'].quantile(0.75),\
        df_train['TotalPricePerMonth'].quantile(1)]

labels = [0, 1, 2, 3]

df_score_cat_train = pd.cut(df_train['TotalPricePerMonth'], bins=bins, labels=labels)

bins = [-np.inf,df_test['TotalPricePerMonth'].quantile(0.25),\
        df_test['TotalPricePerMonth'].quantile(0.50),\
        df_test['TotalPricePerMonth'].quantile(0.75),\
        df_test['TotalPricePerMonth'].quantile(1)]

labels = [0, 1, 2, 3]

df_score_cat_test = pd.cut(df_test['TotalPricePerMonth'], bins=bins, labels=labels)

# Generation of Bag Of Words labels and columns
print('Generation of Bag Of Words labels and columns')

df_train = df_train_ori
df_test = df_test_ori

preparation_pipeline = Pipeline([
    #('features_selector', FeaturesSelector(features_toselect=MODEL_FEATURES)),
    ('bow_encoder', BowEncoder()),
    ('agregate_to_client_level', AgregateToClientLevel(top_value_products, compute_rfm=True)),
    ('features_selector', FeaturesSelector(features_toselect=['DescriptionNormalized', 'RfmScore'])),
    #('scaler', LogScalerMultiple(features_toscale=['TotalPricePerMonth', 'TotalQuantityPerMonth'])),

    # Faire la réduction dimensionnelle à part pour les bag of words et pour les autres features

    #('minmaxscaler', MinMaxScalerMultiple(features_toscale=['RfmScore'])),
    ('dimensionality_reductor', DimensionalityReductor(features_totransform=['DescriptionNormalized'], \
                                                        algorithm_to_use='NCA', n_dim=1, labels_featurename='RfmScore')),
    ('minmaxscaler_final', MinMaxScalerMultiple(features_toscale='ALL_FEATURES')),
])

df_train = preparation_pipeline.fit_transform(df_train)
df_test = preparation_pipeline.transform(df_test)

print('End of preparation pipeline transform')

kmeans_per_k = [KMeans(n_clusters=k, random_state=42).fit(df_train.loc[:, 0].to_numpy().reshape(-1,1))
                for k in range(1, 50)]

labels_test_per_k = [model.predict(df_test.loc[:, 0].to_numpy().reshape(-1,1)) for model in kmeans_per_k[1:]]

silhouette_scores = [silhouette_score(df_train.loc[:, 0].to_numpy().reshape(-1,1), model.labels_)
                     for model in kmeans_per_k[1:]]

silhouette_scores_test = [silhouette_score(df_test.loc[:, 0].to_numpy().reshape(-1,1), labels_test) for labels_test in labels_test_per_k]

entropy_mean_score_per_k_train = []

for model in kmeans_per_k[1:]:
    unique_labels = np.unique(model.labels_)

    entropy_sum = 0
    for unique_label in unique_labels:
        entropy_sum += entropy(df_train['RfmScore'][model.labels_ == unique_label].to_numpy())

    entropy_sum = entropy_sum / len(unique_labels)

    entropy_mean_score_per_k_train.append(entropy_sum)

entropy_mean_score_per_k_test = []

print('Next')

for labels_test in labels_test_per_k:
    unique_labels = np.unique(labels_test)

    entropy_sum = 0
    for unique_label in unique_labels:
        entropy_sum += entropy(df_test['RfmScore'][labels_test == unique_label].to_numpy())

    entropy_sum = entropy_sum / len(unique_labels)

    entropy_mean_score_per_k_test.append(entropy_sum)

# Cluster corresponding to kmeans_per_k[10] has the best silhouette score
bow_labels_train = kmeans_per_k[10].labels_  # sample : [4 7 8 ... 4 8 1]
bow_labels_test = kmeans_per_k[10].predict(df_test.loc[:, 0].to_numpy().reshape(-1,1))

series_bow_labels_train = pd.Series(bow_labels_train)
series_bow_labels_test = pd.Series(bow_labels_test)

series_bow_labels_train.set_axis(df_train.index, axis=0, inplace=True)
series_bow_labels_test.set_axis(df_test.index, axis=0, inplace=True)

rfm_score_train = df_train['RfmScore']
rfm_score_test = df_test['RfmScore']


import csv,os
from Common.formatting import *
from Common.auxillary_functions import *

ROOT = 'C:\\Users\\acesp\\Desktop\\Site Survey Responses\\CSVs/'
base_files = ('Moreton (Avaya - 1-100)','Moreton (Avaya - 101+)','Winters (Avaya - 1-100)','Winters (Avaya - 101+)','Moreton (Mix - 1-100)','Moreton (Mix - 101+)','Winters (Mix - 1-100)','Winters (Mix - 101+)','Self','Other','Avaya - No Core')
PATHS = [ROOT+'Tele Core Sites/'+file+'.csv' for file in base_files]
SITES = ROOT+'Site Summary 100418 v4.csv'
GATEWAYS = ROOT+'Site Status 100518 v1.csv'
tSize = ('XXS','XS','S','M','L','XL','XXL')
moreton = ('moreton core','remote moreton','moreton')
winters = ('winters core','remote winters','winters')
ADD_CISCO_NUMBERS = [0,0,0,0,1,1,1,1,1,1,0]
cisco = ('C','Ccc','C+Ccc','')
avaya = ('A','Acc','A+Acc')

def init_files(paths, add_cisco):
	for path in paths:
		open(path,'w').close()
	files = [open(path, 'a') for path in PATHS]
	for i in range(len(files)):
		if add_cisco[i]:
			files[i].write('\"Site ID\",\"Address\",\"t-Shirt\",\"Total Avaya Business Users\",\"Total Avaya CC Agents\",\"Total Cisco Business Users\",\"Total Cisco CC Agents\",G430,G450,G650,G700,\"Total Avaya Users\",\"Total Cisco Users\",\"POE Capability\"')
		else:
			files[i].write('\"Site ID\",\"Address\",\"t-Shirt\",\"Total Avaya Business Users\",\"Total Avaya CC Agents\",G430,G450,G650,G700,\"Total Users\",\"POE Capability\"')
	return files

def get_gateways(path):
	with open(path, newline='') as file:
		reader = csv.reader(file, delimiter=',', quotechar='\"')
		return {(str(row[0]).lower(), row[1].lower()):row[19:23] for row in reader}

def get_total_users_agents(site):
	ua = [0,0,0,0]
	comp = ('basic','advanced')
	for program in site.programs:
		if program.telephony_users.platform == 'avaya':
			ua[0] += program.telephony_users.total
		elif program.telephony_users.platform == 'cisco':
			ua[2] += program.telephony_users.total
		if program.cc_complexity.complexity in comp:
			if program.contact_center.platform == 'avaya':
				ua[1] += program.contact_center.total
			elif program.contact_center.platform == 'cisco':
				ua[3] += program.contact_center.total
	return ua

def get_base_critera(site, combo):
	index = 0
	if combo in cisco:	return ''

	if site.avaya_type in moreton:
		if combo in avaya:	index = 0
		else:	index = 2
	elif site.avaya_type in winters:
		if combo in avaya:	index = 1
		else:	index = 3
	elif site.avaya_type == 'self':
		return 8
	else:
		if combo in avaya:	return 10
		else:	return 9

	if site.site_size<4:	return index*2
	else:	return index*2+1

def refine_ua(ua, combo):
	return [x if x else '' for x in ua]

def add_to_file(site, gateways, files):
	combo = determine_platform_combination(site)
	criteria = get_base_critera(site, combo)
	ua = get_total_users_agents(site)
	tshirt = tSize[site.site_size]
	gtw = ['','','','']
	group = [site.id, '\"'+site.addr+'\"', tshirt]

	temp = str(site.id).split('.')[0]
	key = (temp.lower(), site.addr.lower())

	if criteria=='':	return
	if key in gateways:
		gtw = gateways[key]
	else:
		print('issue:', str(site.id), 'not in gateways list')

	if criteria in (0,1,2,3,10):
		group += refine_ua(ua[:2], combo) + gtw + [sum(ua[:2]), site.poe_capable]
	else:
		group += refine_ua(ua, combo) + gtw + [sum(ua[:2]), sum(ua[2:]), site.poe_capable]
	output = ','.join([str(g) for g in group])
	files[criteria].write('\n'+output)

def user_summary():
	files = init_files(PATHS, ADD_CISCO_NUMBERS)
	sites = load_data(SITES)
	sites = make_classes(sites)
	gateways = get_gateways(GATEWAYS)
	for site in sites:	add_to_file(site, gateways, files)
	for file in files:	file.close()
	print('done')

if __name__ == '__main__':
	user_summary()


#!/usr/bin/python3

import os.path
import unittest

import loader

import devpipeline.config.override
import devpipeline.config.paths

_config_dir = "{}/../files".format(os.path.dirname(os.path.abspath(__file__)))

class TestConfigOverride(unittest.TestCase):
    def _validate(self, expected, override, actual):
        self.assertTrue(override in expected)
        expected_overrides = expected[override]
        self.assertEqual(len(expected_overrides), len(actual))
        for key, value in expected_overrides.items():
            self.assertTrue(key in actual)
            self.assertEqual(value, actual[key])

    def test_empty(self):
        def _dont_call(override, values):
            raise Exception("Shouldn't have been called")

        count = devpipeline.config.override.read_all_overrides(
            devpipeline.config.paths.get_overrides_root(_config_dir),
            ["empty"], "foo", _dont_call)
        self.assertEqual(0, count)

    def test_append(self):
        expected = {
            "simple": {
                "val.append": "abc",
                "lav.append": "xyz"
            }
        }
        count = devpipeline.config.override.read_all_overrides(
            devpipeline.config.paths.get_overrides_root(_config_dir),
            ["simple"], "foo",
            lambda override, vals:
                self._validate(expected, override, vals))
        self.assertEqual(1, count)

    def test_multiappend(self):
        expected = {
            "simple": {
                "val.append": "abc",
                "lav.append": "xyz"
            },
            "trivial": {
                "val.append": "ijk"
            }
        }
        count = devpipeline.config.override.read_all_overrides(
            devpipeline.config.paths.get_overrides_root(_config_dir),
            ["simple", "trivial"], "foo",
            lambda override, vals:
                self._validate(expected, override, vals))
        self.assertEqual(2, count)

if __name__ == "__main__":
    unittest.main()


import numpy as np
import pytest
from sympy.abc import a, b, c, d, e
import time

from conftest import skipif
from devito.tools import toposort, filter_ordered

pytestmark = skipif(['yask', 'ops'])

@pytest.mark.parametrize('elements, expected', [
    ([[a, b, c], [c, d, e]], [a, b, c, d, e]),
    ([[e, d, c], [c, b, a]], [e, d, c, b, a]),
    ([[a, b, c], [b, d, e]], [a, b, d, c, e]),
    ([[a, b, c], [d, b, c]], [a, d, b, c]),
    ([[a, b, c], [c, d, b]], None),
])
def test_toposort(elements, expected):
    try:
        ordering = toposort(elements)
        assert ordering == expected
    except ValueError:
        assert expected is None

def test_sorting():
    key = lambda x: x
    array = np.random.rand(10000)
    t0 = time.time()
    sort_key = filter_ordered(array, key=key)
    t1 = time.time()
    sort_nokey = filter_ordered(array)
    t2 = time.time()
    # This one is slightly faster
    assert t2 - t1 < .5 * (t1 - t0)
    assert sort_key == sort_nokey


from opentrons import labware, instruments, robot
import logging
logging.basicConfig(level=logging.DEBUG)

metadata = {
	'protocolName':'NAD kinetics assay',
	'author':'Alex Telfar <te1farac@gmail.com>',
	'source':'kinetics.py'
}

def load_labware():
    wells96 = labware.load('corning_96_wellplate_360ul_flat', '1')
    liquid_input = labware.load('opentrons_10_tuberack_nest_4x50ml_6x15ml_conical', '2')
    tiprack300 = labware.load('tiprack-200ul', '3')

    pipette300 = instruments.P300_Single(
        mount='left',
        tip_racks=[tiprack300])

    # pipette10 = instruments.P10_Single(
    #     mount='right',
    #     tip_racks=[tiprack10])

    return (pipette300,
            # pipette10,
            liquid_input['A3'],  # trash
            liquid_input['A4'],  # buffer source,
            liquid_input['A1'],  # nad source
            liquid_input['A2'],  # enzyme source
            liquid_input['B1'],  # substrate source
            wells96)  # 96 well plate

def serial_dilution(pipette, substrate_source, dilutant_source, output, trash,
    total_mixing_volume, num_of_dilutions, dilution_factor):
    transfer_volume = total_mixing_volume/dilution_factor
    diluent_volume = total_mixing_volume - transfer_volume

    # Distribute diluent across the plate to the the number of samples
    # And add diluent to one column after the number of samples for a blank
    for col in output.cols(*range(2, num_of_dilutions+1)):
        pipette.distribute(total_mixing_volume, dilutant_source, col)

    for row in output.rows():
        pipette.transfer(
            transfer_volume,
            row.wells(*range(1, num_of_dilutions)),
            row.wells(*range(2, num_of_dilutions+1)),
            mix_after=(3, total_mixing_volume / 2))

        pipette.transfer(
            transfer_volume,
            row.wells(num_of_dilutions),
            trash)

def run_custom_protocol(
        dilution_factor: float=2.0,
        num_of_dilutions: int=4,
        replicates: int=6,
        total_mixing_volume: float=200.0,
        substrate_conc: float=1,  # 1M conc
        nad_conc: float=20.0,  # 10mM of NAD
        v_enzyme: float=50.0, # 5uL of enzyme
        ):

    transfer_volume = total_mixing_volume/dilution_factor
    diluent_volume = total_mixing_volume - transfer_volume

    p300, trash, buffer_s, nad_s, enzyme_s, substrate_s, wells96 = load_labware()

    # serial dilution of the substrate
    serial_dilution(p300, substrate_s, buffer_s, wells96, trash, 100, 4, 0.5)

    # add 10mM of NAD to each well
    p300.distribute(nad_conc, nad_s, wells96.wells())

    # add 5uL of enzyme to each well
    p300.distribute(v_enzyme, enzyme_s, wells96.wells())

if __name__ == '__main__' or __name__ == 'builtins':
	run_custom_protocol()


import glob
import os

import numpy as np
import pandas as pd

def get_accuracy_paths(loss_dir, dataset, init, layer):
  prefix = os.path.join(loss_dir, dataset + "_" + init, "layers_" + str(layer))
  suffix = 'acc_test.npy'
  return glob.glob(os.path.join(prefix + '*', suffix))

def build_frame(aggr_func, accuracies, datasets, inits, layers):
  frame = pd.DataFrame(index=layers)
  for dataset in datasets:
    for init in inits:
      name = dataset + "_" + init
      values = []
      for layer in layers:
        values.append(aggr_func(accuracies[dataset][init][layer]))
      frame[name] = values

  return frame

def get_frames():
    loss_dir = 'losses'
    datasets = ['citeseer', 'cora', 'pubmed']
    inits = ['init', 'no_init']
    layers = [2, 4, 8, 10, 16]

    accuracies  = {}
    for dataset in datasets:
      accuracies[dataset] = {}
      for init in inits:
        accuracies[dataset][init] = {}
        for layer in layers:
          accuracies[dataset][init][layer] = []
          paths = get_accuracy_paths(loss_dir, dataset, init, layer)
          for path in paths:
            accuracies[dataset][init][layer].append(np.load(path))
    functions = [np.max, np.mean, np.median, np.std]
    frames = []
    for func in functions:
      frames.append(build_frame(func, accuracies, datasets, inits, layers))
    return frames
def main():
  func_names = ['Max:', 'Average:', 'Median:', 'Std:']
  frames = get_frames()
  for i in range(len(func_names)):
    print(func_names[i])
    print(frames[i])

if __name__ == '__main__':
  main()


from collections import defaultdict

def log_missing():
    print('key added')
    return 0

def sub_gen(i=10):
    for x in range(i):
        yield x

    print('done')
    return 'finished'

ev = None

def deleg():
    global ev
    ev = yield from sub_gen(5)

def test2():
    d = deleg()
    for i in d:
        print(i)
    print(ev)

def test():
    current =  {'green' : 12, 'red' :3}
    increcement  = [('blue', 2), ('black', 6), ('red', 66)]
    res = defaultdict(log_missing, current)

    print(current)
    for k, v in increcement:
        res[k] += v
    print(res)

if __name__ == '__main__':
    test2()


from sys import argv
import ConfigParser
import send_request
import email_tools

# date_from/to should be a timestamp in ISO 8601 format: (YYYY-MM-DDTHH:MM:SSZ)
script, date_from, date_to = argv

Config = ConfigParser.ConfigParser()
Config.read("src/config.ini")  # Note that this causes errors if it's called from anywhere other than repo root

user = Config.get("settings", "repo-owner")
repo = Config.get("settings", "repo")
paths = Config.options("file_paths")
access_token = Config.get("settings", "access_token")

# Go and find all commits in the given timeframe for the given file-paths.
list_of_request_outputs = []
for path in paths:
    url = ("https://api.github.com/repos" +
           "/" + user + "/" + repo + "/commits" +
           "?path=" + Config.get("file_paths", path) +
           "&since=" + date_from +
           "&until=" + date_to)

    json_output = send_request.request_url(url)
    list_of_request_outputs.append(json_output)

# Get all of the individual commit URL's that have modified the tracked files
list_of_commit_urls = []
for request_output in list_of_request_outputs:
    for json in request_output:
        list_of_commit_urls.append(json["url"])

# Go through the individual commits that we want to send security emails about
commits_to_be_emailed = []
for commit_url in list_of_commit_urls:
    commit_json = send_request.request_url(commit_url)
    commits_to_be_emailed.append({"Name": commit_json["commit"]["committer"]["name"],
                                  "Message": commit_json["commit"]["message"],
                                  "Files": commit_json["files"],
                                  "Url": commit_json["html_url"]})

# Send an email for the individual commits that have modified the tracked files
for commit in commits_to_be_emailed:
    email = email_tools.create_email(commit)
    email_tools.send_email(email, commit)


"""Decorator to handle vectorization.

This abstracts the backend type.
"""

import math

import geomstats.backend as gs

POINT_TYPES = ["scalar", "vector", "matrix"]
FLEXIBLE_TYPE = "point"
OTHER_TYPES = ["point_type", "else"]

POINT_TYPES_TO_NDIMS = {"scalar": 2, "vector": 2, "matrix": 3}

ERROR_MSG = "Invalid type: %s."

def _get_max_ndim_point(*point):
    """Identify point with higher dimension.

    Parameters
    ----------
    point : array-like

    Returns
    -------
    max_ndim_point : array-like
        Point with higher dimension.
    """
    max_ndim_point = point[0]
    for point_ in point[1:]:
        if point_.ndim > max_ndim_point.ndim:
            max_ndim_point = point_

    return max_ndim_point

def get_n_points(space, *point):
    """Compute the number of points.

    Parameters
    ----------
    space : Manifold object
        Space to which point belongs.
    point : array-like
        Point belonging to the space.

    Returns
    -------
    n_points : int
        Number of points.
    """
    point_max_ndim = _get_max_ndim_point(*point)

    if space.point_ndim == point_max_ndim.ndim:
        return 1

    return math.prod(point_max_ndim.shape[: -space.point_ndim])

def check_is_batch(space, *point):
    """Check if inputs are batch.

    Parameters
    ----------
    space : Manifold object
        Space to which point belongs.
    point : array-like
        Point belonging to the space.

    Returns
    -------
    is_batch : bool
        Returns True if point contains several points.
    """
    return any(point_.ndim > space.point_ndim for point_ in point)

def get_batch_shape(space, *point):
    """Get batch shape.

    Parameters
    ----------
    space : Manifold
        Space to which point belongs.
    point : array-like or None
        Point belonging to the space.

    Returns
    -------
    batch_shape : tuple
        Returns the shape related with batch. () if only one point.
    """
    point = list(filter(_is_not_none, point))
    if len(point) == 0:
        return ()
    point_max_ndim = _get_max_ndim_point(*point)
    return point_max_ndim.shape[: -space.point_ndim]

def repeat_point(point, n_reps=2, expand=False):
    """Repeat point.

    Parameters
    ----------
    point : array-like
        Point of a space.
    n_reps : int
        Number of times the point should be repeated.
    expand : bool
        Repeat even if n_reps == 1.

    Returns
    -------
    rep_point : array-like
        point repeated n_reps times.
    """
    if not expand and n_reps == 1:
        return gs.copy(point)

    return gs.repeat(gs.expand_dims(point, 0), n_reps, axis=0)

def _is_not_none(value):
    """Check if a value is None."""
    return value is not None

def repeat_out(space, out, *point, out_shape=()):
    """Repeat out shape after finding batch shape.

    Parameters
    ----------
    space : Manifold
        Space to which point belongs.
    out : array-like
        Output to be repeated
    point : array-like or None
        Point belonging to the space.
    out_shape : tuple
        Indicates out shape for no batch computations.

    Returns
    -------
    out : array-like
        If no batch, then input is returned. Otherwise it is broadcasted.
    """
    point = filter(_is_not_none, point)
    batch_shape = get_batch_shape(space, *point)
    if out.shape[: -len(out_shape)] != batch_shape:
        return gs.broadcast_to(out, batch_shape + out_shape)
    return out

def decorator(input_types):
    """Vectorize geomstats functions.

    This decorator assumes that its function:

    - works with fully-vectorized inputs,
    - returns fully-vectorized outputs,

    where "fully-vectorized" means that:

    - one scalar has shape [1, 1],
    - n scalars have shape [n, 1],
    - one d-D vector has shape [1, d],
    - n d-D vectors have shape [n, d],etc
    etc.

    The decorator:

    - gets the types of all inputs of its function:
        - args,
        - kwargs,
        - optional kwargs,
            - e.g. input_type=None,
    - gets the type of the output of its function,
        - e.g. distinguishes between 1D 'vector' vs 'scalar',
    - gets the initial shapes of all inputs of its function,
    - if needed, adapts the types of the inputs,
        - e.g. distinguishes between 'vector' or 'matrix' inputs,
        using variables 'point_type' or 'default_point_type',
    - converts the inputs into fully-vectorized inputs,
    - calls the function,
    - adapts the output shapes to match the users' expectations,
      using the initial shapes of the inputs.

    Parameters
    ----------
    input_types : list
        List of inputs' input_types, including for optional inputs.
        The `input_type`s of optional inputs will not be read
        by the decorator if the corresponding input is not given.
    """
    if not isinstance(input_types, list):
        input_types = list(input_types)

    def aux_decorator(function):
        def wrapper(*args, **kwargs):
            args_types, kwargs_types, opt_kwargs_types, is_scal = get_types(
                input_types, args, kwargs
            )
            args_types, kwargs_types, kwargs = adapt_types(
                args_types, kwargs_types, opt_kwargs_types, args, kwargs
            )
            args_kwargs_types = args_types + kwargs_types

            args_shapes = get_initial_shapes(args_types, args)
            kwargs_shapes = get_initial_shapes(kwargs_types, kwargs.values())
            initial_shapes = args_shapes + kwargs_shapes

            vect_args = vectorize_args(args_types, args)
            vect_kwargs = vectorize_kwargs(kwargs_types, kwargs)

            result = function(*vect_args, **vect_kwargs)

            result = adapt_result(result, initial_shapes, args_kwargs_types, is_scal)

            return result

        return wrapper

    return aux_decorator

def get_types(input_types, args, kwargs):
    """Extract the types of args, kwargs, optional kwargs and output.

    Parameters
    ----------
    input_types : list
        List of inputs' input_types, including for optional inputs.
    args : tuple
        Args of a function.
    kwargs : dict
        Kwargs of a function.

    Returns
    -------
    args_types : list
        Types of args.
    kwargs_types : list
        Types of kwargs.
    opt_kwargs_types : list
        Types of optional kwargs.
    is_scal : bool
        Boolean determining if the output is a scalar.
    """
    len_args = len(args)
    len_kwargs = len(kwargs)
    len_total = len_args + len_kwargs

    args_types = input_types[:len_args]
    kwargs_types = input_types[len_args:len_total]

    opt_kwargs_types = []
    is_scal = True
    if len(input_types) > len_total:
        opt_kwargs_types = input_types[len_total:]
        last_input_type = input_types[-1]
        if "output_" in last_input_type and last_input_type != "output_scalar":
            is_scal = False
            opt_kwargs_types = input_types[len_total:-1]
    return (args_types, kwargs_types, opt_kwargs_types, is_scal)

def adapt_types(args_types, kwargs_types, opt_kwargs_types, args, kwargs):
    """Adapt the list of input input_types.

    Some functions are implemented with array-like arguments that can be either
    'vector' or 'matrix' depending on the value of the 'point_type'
    argument.

    This function reads the 'point_type' argument, and adapt the actual
    type of the input array-like arguments.

    Parameters
    ----------
    args_types : list
        Types of args.
    kwargs_types : list
        Types of kwargs.
    opt_kwargs_types : list
        Types of optional kwargs.
    args : tuple
        Args of a function.
    kwargs : dict
        Kwargs of a function.

    Returns
    -------
    args_types : list
        Adapted types of args.
    kwargs_types : list
        Adapted types of kwargs.
    """
    in_args = "point_type" in args_types
    in_kwargs = "point_type" in kwargs_types
    in_optional = "point_type" in opt_kwargs_types

    if in_args or in_kwargs or in_optional:
        if in_args:
            i_input_type = args_types.index("point_type")
            input_type = args[i_input_type]
        elif in_kwargs:
            input_type = kwargs["point_type"]

        elif in_optional:
            obj = args[0]
            input_type = obj.default_point_type
            kwargs["point_type"] = input_type
            kwargs_types.append("point_type")

        args_types = [input_type if pt == FLEXIBLE_TYPE else pt for pt in args_types]
        kwargs_types = [
            input_type if pt == FLEXIBLE_TYPE else pt for pt in kwargs_types
        ]
    return args_types, kwargs_types, kwargs

def get_initial_shapes(input_types, args):
    """Extract shapes and ndims of input args or kwargs values.

    Store the shapes of the input args, or kwargs values,
    that are array-like, store None otherwise.

    Parameters
    ----------
    input_types : list
        Point types corresponding to the args, or kwargs values.
    args : tuple or dict_values
        Args, or kwargs values, of a function.

    Returns
    -------
    in_shapes : list
        Shapes of array-like input args, or kwargs values.
    """
    in_shapes = []

    for arg, input_type in zip(args, input_types):
        if input_type == "scalar":
            arg = gs.array(arg)

        if input_type in POINT_TYPES and arg is not None:
            in_shapes.append(gs.shape(arg))
        elif input_type in OTHER_TYPES or arg is None:
            in_shapes.append(None)
        else:
            raise ValueError(ERROR_MSG % input_type)
    return in_shapes

def vectorize_args(input_types, args):
    """Vectorize input args.

    Transform input array-like args into their fully-vectorized form,
    where "fully-vectorized" means that:

    - one scalar has shape [1, 1],
    - n scalars have shape [n, 1],
    - one d-D vector has shape [1, d],
    - n d-D vectors have shape [n, d],
    etc.

    Parameters
    ----------
    input_types : list
        Point types corresponding to the args.
    args : tuple
        Args of a function.

    Returns
    -------
    vect_args : tuple
        Args of the function in their fully-vectorized form.
    """
    vect_args = []
    for arg, input_type in zip(args, input_types):
        if input_type == "scalar":
            vect_arg = gs.to_ndarray(arg, to_ndim=1)
            vect_arg = gs.to_ndarray(vect_arg, to_ndim=2, axis=1)
        elif input_type in POINT_TYPES and arg is not None:
            vect_arg = gs.to_ndarray(arg, to_ndim=POINT_TYPES_TO_NDIMS[input_type])
        elif input_type in OTHER_TYPES or arg is None:
            vect_arg = arg
        else:
            raise ValueError(ERROR_MSG % input_type)
        vect_args.append(vect_arg)
    return tuple(vect_args)

def vectorize_kwargs(input_types, kwargs):
    """Vectorize input kwargs.

    Transform input array-like kwargs into their fully-vectorized form,
    where "fully-vectorized" means that:

    - one scalar has shape [1, 1],
    - n scalars have shape [n, 1],
    - one d-D vector has shape [1, d],
    - n d-D vectors have shape [n, d],
    etc.

    Parameters
    ----------
    input_types :list
        Point types corresponding to the args.
    kwargs : dict
        Kwargs of a function.

    Returns
    -------
    vect_kwargs : dict
        Kwargs of the function in their fully-vectorized form.
    """
    vect_kwargs = {}
    for key_arg, input_type in zip(kwargs.keys(), input_types):
        arg = kwargs[key_arg]
        if input_type == "scalar":
            vect_arg = gs.to_ndarray(arg, to_ndim=1)
            vect_arg = gs.to_ndarray(vect_arg, to_ndim=2, axis=1)
        elif input_type in POINT_TYPES and arg is not None:
            vect_arg = gs.to_ndarray(arg, to_ndim=POINT_TYPES_TO_NDIMS[input_type])
        elif input_type in OTHER_TYPES or arg is None:
            vect_arg = arg
        else:
            raise ValueError(ERROR_MSG % input_type)
        vect_kwargs[key_arg] = vect_arg
    return vect_kwargs

def adapt_result(result, initial_shapes, args_kwargs_types, is_scal):
    """Adapt shape of output.

    This function squeezes the dim 0 or 1 of the output, depending on:

    - the type of the output: scalar vs else,
    - the initial shapes or args and kwargs provided by the user.

    Parameters
    ----------
    result : unspecified
        Output of the function.
    initial_shapes : list
        Shapes of args and kwargs provided by the user.
    args_kwargs_types : list
        Types of args and kwargs.
    is_scal : bool
        Boolean determining if the output 'result' is a scalar.

    Returns
    -------
    result : unspecified
        Output of the function, with adapted shape.
    """
    if squeeze_output_dim_1(result, initial_shapes, args_kwargs_types, is_scal):
        if result.shape[1] == 1:
            result = gs.squeeze(result, axis=1)

    if (
        squeeze_output_dim_0(result, initial_shapes, args_kwargs_types)
        and result.shape[0] == 1
    ):
        result = gs.squeeze(result, axis=0)

    return result

def squeeze_output_dim_0(result, in_shapes, input_types):
    """Determine if the output needs to be squeezed on dim 0.

    The dimension 0 is squeezed iff all input parameters:

    - contain one sample,
    - have the corresponding dimension 0 squeezed,
      i.e. if all input parameters have ndim strictly less than the ndim
      corresponding to their vectorized shape.

    Parameters
    ----------
    in_ndims : list
        Initial ndims of input parameters, as entered by the user.
    input_types : list
        Associated list of input_type of input parameters.

    Returns
    -------
    squeeze : bool
        Boolean deciding whether to squeeze dim 0 of the output.
    """
    if isinstance(result, tuple):
        return False
    if isinstance(result, list):
        return False

    for in_shape, input_type in zip(in_shapes, input_types):
        if input_type not in POINT_TYPES:
            continue
        in_ndim = None
        if in_shape is not None:
            in_ndim = len(in_shape)
        if in_ndim is not None:
            vect_ndim = POINT_TYPES_TO_NDIMS[input_type]
            if in_ndim > vect_ndim:
                raise ValueError(
                    "Fully-vectorizing an input can only increase its ndim."
                )
            if in_ndim == vect_ndim:
                return False
    return True

def squeeze_output_dim_1(result, in_shapes, input_types, is_scal=True):
    """Determine if the output needs to be squeezed on dim 1.

    This happens if the user represents scalars as array of shapes:
    [n_samples,] instead of [n_samples, 1]
    Dimension 1 is squeezed by default if input_type is 'scalar'.
    Dimension 1 is not squeezed if the user inputs at least one scalar with
    a singleton in dimension 1.

    Parameters
    ----------
    result: array-like
        Result output by the function, before reshaping.
    in_shapes : list
        Initial shapes of input parameters, as entered by the user.
    input_types : list
        Associated list of input_type of input parameters.

    Returns
    -------
    squeeze : bool
        Boolean deciding whether to squeeze dim 1 of the output.
    """
    if not is_scal:
        return False
    if not is_scalar(result):
        return False

    for shape, input_type in zip(in_shapes, input_types):
        if input_type == "scalar":
            ndim = len(shape)
            if ndim > 2:
                raise ValueError("The ndim of a scalar cannot be > 2.")
            if ndim == 2:
                return False
    return True

def is_scalar(vect_array):
    """Test if a "fully-vectorized" array represents a scalar.

    Parameters
    ----------
    vect_array :  array-like
        Array to be tested.

    Returns
    -------
    is_scalar : bool
        Boolean determining if vect_array is a fully-vectorized scalar.
    """
    if isinstance(vect_array, tuple):
        return False
    if isinstance(vect_array, list):
        return False

    has_ndim_2 = vect_array.ndim == 2
    if not has_ndim_2:
        return False
    has_singleton_dim_1 = vect_array.shape[1] == 1
    return has_singleton_dim_1


import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
import time
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from extract_feature import *
# NOTE: the next import is only valid for scikit-learn version <= 0.17
# for scikit-learn >= 0.18 use:
# from sklearn.model_selection import train_test_split
from sklearn.cross_validation import train_test_split
import pickle

# Read in cars and notcars
cars = glob.glob('./train_data/vehicles/*/*.png'); train_data_tpye = 'png'; notcars = glob.glob('./train_data/non-vehicles/*/*.png')
#cars = glob.glob('./hog_test_imgs/vehicles_smallset/*/*.jpeg'); train_data_tpye = 'jpeg'; #notcars = glob.glob('./hog_test_imgs/non-vehicles_smallset/*/*.jpeg')
sample_size = None
cars = cars[0:sample_size]
notcars = notcars[0:sample_size]

color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9  # HOG orientations
pix_per_cell = 8 # HOG pixels per cell
cell_per_block = 2 # HOG cells per block
hog_channel = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_size = (32, 32) # Spatial binning dimensions
hist_bins = 32    # Number of histogram bins
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
hog_feat = True # HOG features on or off
y_start_stop = [None, None] # Min and max in y to search in slide_window()

t = time.time()
print("start extract car_features")
car_features = extract_features(cars, train_data_tpye, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)
print("start extract notcar_features")
notcar_features = extract_features(notcars, train_data_tpye, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)

X = np.vstack((car_features, notcar_features)).astype(np.float64)

# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))

# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=rand_state)

t2 = time.time()
print(round(t2-t, 2), 'Seconds to extract features SVC...')

print("X_train shape: {} \X_test shape: {}".format(X_train.shape, X_test.shape))

# Fit a per-column scaler
X_scaler = StandardScaler().fit(X_train)
# Apply the scaler to X
X_train = X_scaler.transform(X_train)
X_test = X_scaler.transform(X_test)

print('Using:',orient,'orientations',pix_per_cell,
    'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC
svc = LinearSVC()
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')

# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))

# pickle SVC
pickle_file = 'svc_acc_%f.p'%round(svc.score(X_test, y_test), 4)
try:
    with open(pickle_file, 'wb') as pfile:
        pickle.dump(
                {
                    'svc': svc,
                    'scaler': X_scaler,
                    'color_space': color_space,
                    'orient': orient,
                    'pix_per_cell': pix_per_cell,
                    'cell_per_block': cell_per_block,
                    'spatial_size': spatial_size,
                    'hist_bins': hist_bins,
                    'spatial_feat': spatial_feat,
                    'hist_feat': hist_feat,
                },
                pfile, pickle.HIGHEST_PROTOCOL)
except Exception as e:
    print('Unable to save data to', pickle_file, ':', e)
    raise


from dateutil import rrule
from datetime import datetime, timedelta

import workout          # Entity
import term_io as io    # Interface class
import xlsx_io as file  # Storage

# Initalize which exercises to track, their current weights and planned increments
def initWorkout(wo):
    # Give the new program a name
    wo.workoutName = io.reqInput("Enter program name: ")
    setExercises(wo)
    setWeights(wo)
    setIncrements(wo)

def modifyWorkout(wo):
    choice = ''
    while(choice != '4'):
        io.printStatus(wo)
        options = {}
        options['1'] = "Set exercises"
        if(len(wo.ex) > 0):
            options['2'] = "Set weights"
            options['3'] = "Set increments"
        options['4'] = "Continue"
        choice = io.chooseAction(options)

        if(choice == '1'):
            setExercises(wo)
        elif(choice == '2'):
            setWeights(wo)
        elif(choice == '3'):
            setIncrements(wo)

# Name exercises that you want to track
def setExercises(wo):
    choice = ''
    while(choice != '3'):
        if(len(wo.ex) > 0):
            io.ssprint("Exercises:")
            io.printList(wo.ex)
        else:
            io.ssprint("No exercises set")
        options = {}
        options['1'] = "Add an exercise"
        options['2'] = "Remove an exercise"
        options['3'] = "Done"
        choice = io.chooseAction(options)

        if(choice == '1'):
            ex = io.reqInput("Exercise name: ")
            if(ex != ""):               # If anything was returned
               wo.ex.append(ex)
        elif(choice == '2'):
            i = 0
            exerciseOptions = {}
            for ex in wo.ex:
                i = i+1
                exerciseOptions[str(i)] = ex
            io.ssprint("Enter the number of the exercise you want to remove:")
            exerciseChoice = io.chooseAction(exerciseOptions)

            removee = exerciseOptions[exerciseChoice]
            if(io.confirmRemoval(removee)):
               wo.ex.remove(removee)

# Set the current weight of each exercise
def setWeights(wo):
    if(len(wo.ex) > 0):
        choice = ''
        while(choice != '-1'):       #Arbritrary number. Loop should exit on return command
            if(len(wo.weights) > 0):
                io.ssprint("Weights:")
                io.printDict(wo.weights)
            else:
                io.ssprint("No weights set")

            io.ssprint("Choose an exercise to set your working weight:")
            options = {}
            options['1'] = "All"
            i = 2                    # Start i as next available option
            for ex in wo.ex:
                options[str(i)] = ex # Add option
                i = i+1              # Increment i for next option

            doneOpNum = i            # The done option comes after the dynamic middle, so make sure we remember it's number
            options[str(doneOpNum)] = "Done"

            choice = io.chooseAction(options)

            if(choice == '1'):
                io.ssprint("Enter your current weight for each exercise: ")
                wo.weights = io.getDict(wo.ex)
            elif(choice == str(doneOpNum)):
                return
            else:
                exercise = options[choice]                  # Get the name of the exercise chosen
                newWeight = io.reqInput(exercise + ": ")    # Request the new weight
                wo.weights[exercise] = newWeight            # Set the new weight

# Set the planned increment of each exercise
def setIncrements(wo):
    if(len(wo.ex) > 0):
        choice = ''
        while(choice != '-1'):  #Arbritrary number. Loop should exit on return command
            if(len(wo.increments) > 0):
                io.ssprint("Increments:")
                io.printDict(wo.increments)
            else:
                io.ssprint("No increments set")
            io.ssprint("Choose an exercise to set your planned weight increments per workout:")
            options = {}
            options['1'] = "All"
            i = 2                    # Start i as next available option
            for ex in wo.ex:
                options[str(i)] = ex # Add option
                i = i+1              # Increment i for next option

            doneOpNum = i            # The done option comes after the dynamic middle, so make sure we remember it's number
            options[str(doneOpNum)] = "Done"

            choice = io.chooseAction(options)

            if(choice == '1'):
                io.ssprint("Enter your current per workout weight increment for each exercise: ")
                wo.increments = io.getDict(wo.ex)
            elif(choice == str(doneOpNum)):
                return
            else:
                exercise = options[choice]                  # Get the name of the exercise chosen
                newWeight = io.reqInput(exercise + ": ")    # Request the new increment
                wo.weights[exercise] = newWeight            # Set the new increment

def export_xlsx(wo):
    f = file.xlsx_io(wo.workoutName) # Create an instance of our file class

    columns = wo.ex[:]               # Copy wo.ex by value
    columns.insert(0, "Day")
    columns.insert(0, "Date")
    f.setColumns(columns)

    weights = wo.weights
    now = datetime.now()
    for d in rrule.rrule(rrule.DAILY, dtstart=now, until=now+timedelta(days=90)):   # Iterate through 30 days
        if(d.strftime('%a') in ["Mon", "Wed", "Fri"]):                              # If the day is in the list
            row = {}
            row["Day"] = d.strftime('%a')
            row["Date"] = d.strftime('%b %d')
            for ex in wo.ex:
                row[ex] = float(weights[ex])
                weights[ex] = float(weights[ex]) + float(wo.increments[ex])

            f.writeRow(row)

    f.generateXLSX()                # Write the file (Overwrites existing)

def main():
    wo = workout.Workout()
    # TODO: Attempt to load workout
    if(len(wo.ex) <= 0):
        initWorkout(wo)
    modifyWorkout(wo)

    export_xlsx(wo)

main()


# Create a :class:`pyvista.Actor` and assign properties to it.
#
import pyvista as pv
actor = pv.Actor()
actor.prop = pv.Property(
    color='r',
    show_edges=True,
    interpolation='Physically based rendering',
    metallic=0.5,
    roughness=0.1
)
#
# Visualize how the property would look when applied to a mesh.
#
actor.prop.plot()
#
# Set custom properties not directly available in
# :func:`pyvista.Plotter.add_mesh`. Here, we set diffuse, ambient, and
# specular power and colors.
#
pl = pv.Plotter()
actor = pl.add_mesh(pv.Sphere())
prop = actor.prop
prop.diffuse = 0.6
prop.diffuse_color = 'w'
prop.ambient = 0.3
prop.ambient_color = 'r'
prop.specular = 0.5
prop.specular_color = 'b'
pl.show()


#!/usr/bin/env python
# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Finetuning a 🤗 Flax Transformers model for sequence classification on GLUE."""
import argparse
import logging
import math
import os
import random
import sys
from typing import Tuple, Union

import datasets
from dataclasses import dataclass, field
from datasets import load_dataset, load_metric
from torch.utils.data.dataloader import DataLoader
from tqdm.auto import tqdm
from typing import Optional, List
from functools import partial

import transformers
from transformers import (
    BertConfig,
    FlaxBertForSequenceClassification,
    BertTokenizerFast,
    TrainingArguments,
    HfArgumentParser,
    set_seed,
    TensorType
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process
from transformers.utils import logging as hf_logging

from flax import optim, jax_utils, struct
import jax
import jax.numpy as jnp
from flax.training.common_utils import shard, onehot
from flax import linen as nn

class Glue(struct.PyTreeNode):
  sentences: Tuple[str, Union[str, None]] = struct.field(pytree_node=False)
  num_labels: int = struct.field(pytree_node=False)
  is_regression: bool = struct.field(pytree_node=False)

glue_tasks = {
    "cola": Glue(("sentence", None), 2, False),
    "mnli": Glue(("premise", "hypothesis"), 3, False),
    "mnli-mm": Glue(("premise", "hypothesis"), 3, False),
    "mrpc": Glue(("sentence1", "sentence2"), 2, False),
    "qnli": Glue(("question", "sentence"), 2, False),
    "qqp": Glue(("question1", "question2"), 2, False),
    "rte": Glue(("sentence1", "sentence2"), 2, False),
    "sst2": Glue(("sentence", None), 2, False),
    "stsb": Glue(("sentence1", "sentence2"), 1, True),
    "wnli": Glue(("sentence1", "sentence2"), 2, False),
}

@dataclass
class FlaxTrainingArguments(TrainingArguments):
    """We use this to override some defaults in TrainingArguments."""
    do_train: bool = field(default=True, metadata={"help": "Whether to run training."})
    do_eval: bool = field(default=True, metadata={"help": "Whether to run eval on the dev set."})
    do_predict: bool = field(default=True, metadata={"help": "Whether to run predictions on the test set."})

@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.

    Using `HfArgumentParser` we can turn this class
    into argparse arguments to be able to specify them on
    the command line.
    """

    task_name: str = field(
        metadata={
            "help": "The name of the task to train on: " + ", ".join(glue_tasks.keys())
        },
    )
    max_seq_length: int = field(
        default=128,
        metadata={
            "help": "The maximum total input sequence length after tokenization. Sequences longer "
            "than this will be truncated, sequences shorter will be padded."
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
    )

    def __post_init__(self):
        self.task_name = self.task_name.lower()

@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
    """

    model_name_or_path: str = field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None,
        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
    )

def rename_mnli_task(task_name, split):
    # The MNLI task consists of two tasks: MNLI-matched and MNLI-mismatched,
    # which we identify with resp. `mnli` and `mnli-mm` in this script.
    # Both tasks have the same training set, but a different validation and test
    # set. In the `datasets` library, the MNLI-matched validation and test sets
    # are in `mnli_matched`, and those of MNLI-mismatched are in
    # `mnli_mismatched`. Therefore we perform the following rewrites to make
    # sure we obtain the correct train, validation and test sets:
    #
    # `task_name` | `datasets` train set | `dataesets` validation/test set
    # ------------|----------------------|--------------------------------
    # mnli        | mnli                 | mnli_matched
    # mnli-mm     | mnli                 | mnli_mismatched
    if not task_name.startswith("mnli"):
        return task_name
    if split == "train":
        return "mnli"
    elif task_name == "mnli":
        return "mnli_matched"
    else:
        return "mnli_mismatched"

def get_dataset(task_name, split, cache_dir, tokenizer, max_seq_length):
    dataset = load_dataset("glue", rename_mnli_task(task_name, split), split=split, cache_dir=cache_dir)

    sentence1, sentence2 = glue_tasks[task_name].sentences
    def tokenize_fn(examples):
        args = (examples[sentence1],)
        args += () if sentence2 is None else (sentence2,)
        return tokenizer(*args, max_length=max_seq_length, padding="max_length", truncation=True)

    cols_to_remove = dataset.column_names
    cols_to_remove.remove("label")  # Keep the label.
    dataset = dataset.map(tokenize_fn, remove_columns=cols_to_remove, batched=True)
    return dataset

def create_optimizer(model, learning_rate):

    # def adam_optimizer(weight_decay):
    #     return optim.Adam(learning_rate=learning_rate, beta1=0.9,
    #         beta2=0.999, eps=1e-6, weight_decay=weight_decay)

    # optimizer_decay_def = adam_optimizer(weight_decay=0.01)
    # optimizer_no_decay_def = adam_optimizer(weight_decay=0.0)
    # decay = optim.ModelParamTraversal(lambda path, _: 'bias' not in path)
    # no_decay = optim.ModelParamTraversal(lambda path, _: 'bias' in path)
    # optimizer_def = optim.MultiOptimizer(
    #   (decay, optimizer_decay_def), (no_decay, optimizer_no_decay_def))
    optimizer_def = optim.Adam(learning_rate=learning_rate)
    optimizer = optimizer_def.create(model)
    return optimizer

def regression_loss_fn(logits, labels):
    return jnp.mean((logits[..., 0] - labels) ** 2)

def classification_loss_fn(logits, labels):
    logits = nn.log_softmax(logits)
    return -jnp.mean(jnp.sum(onehot(labels, glue_task.num_labels) * logits, axis=-1))

def train_step(optimizer, batch, dropout_rng, apply_fn, glue_loss_fn, lr_scheduler_fn):
    print("Compiling train")
    dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)
    targets = batch.pop("label")

    def loss_fn(params):
        logits = apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=False)[0]
        loss = glue_loss_fn(logits, targets)
        return loss, logits

    step = optimizer.state.step
    lr = lr_scheduler_fn(step)
    grad_fn = jax.value_and_grad(loss_fn, has_aux=True)
    (loss, logits), grad = grad_fn(optimizer.target)
    grad = jax.lax.pmean(grad, "batch")
    # clip_grad_norm = 1.0
    # if clip_grad_norm is not None:
    #   grad_norm = sum([jnp.sum(x ** 2) for x in jax.tree_leaves(grad)])
    #   metrics['grad_norm'] = grad_norm
    #   grad_scale = jnp.where(
    #       grad_norm < clip_grad_norm, 1.0, clip_grad_norm / grad_norm)
    #   grad = jax.tree_map(lambda x: x * grad_scale, grad)
    optimizer = optimizer.apply_gradient(grad, learning_rate=lr)
    metrics = {'loss': loss}
    return optimizer, metrics, new_dropout_rng

def create_learning_rate_scheduler(
        factors='constant * linear_warmup * rsqrt_decay',
        base_learning_rate=0.5,
        warmup_steps=1000,
        decay_factor=0.5,
        steps_per_decay=20000,
        steps_per_cycle=100000):
    """Creates learning rate schedule.
    Interprets factors in the factors string which can consist of:
    * constant: interpreted as the constant value,
    * linear_warmup: interpreted as linear warmup until warmup_steps,
    * rsqrt_decay: divide by square root of max(step, warmup_steps)
    * decay_every: Every k steps decay the learning rate by decay_factor.
    * cosine_decay: Cyclic cosine decay, uses steps_per_cycle parameter.
    * linear_decay: Linear decay, uses steps_per_cycle parameter.
    Args:
      factors: a string with factors separated by '*' that defines the schedule.
      base_learning_rate: float, the starting constant for the lr schedule.
      warmup_steps: how many steps to warm up for in the warmup schedule.
      decay_factor: The amount to decay the learning rate by.
      steps_per_decay: How often to decay the learning rate.
      steps_per_cycle: Steps per cycle when using linear/cosine decay.
    Returns:
      a function learning_rate(step): float -> {'learning_rate': float}, the
      step-dependent lr.
    """
    factors = [n.strip() for n in factors.split('*')]

    def step_fn(step):
        """Step to learning rate function."""
        ret = 1.0
        for name in factors:
            if name == 'constant':
                ret *= base_learning_rate
            elif name == 'linear_warmup':
                ret *= jnp.minimum(1.0, step / warmup_steps)
            elif name == 'rsqrt_decay':
                ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
            elif name == 'rsqrt_normalized_decay':
                ret *= jnp.sqrt(warmup_steps)
                ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
            elif name == 'decay_every':
                ret *= (decay_factor**(step // steps_per_decay))
            elif name == 'cosine_decay':
                progress = jnp.maximum(0.0,
                                      (step - warmup_steps) / float(steps_per_cycle))
                ret *= jnp.maximum(0.0,
                                   0.5 * (1.0 + jnp.cos(jnp.pi * (progress % 1.0))))
            elif name == 'linear_decay':
                progress = jnp.maximum(0.0,
                                      (step - warmup_steps) / float(steps_per_cycle))
                ret *= 1.0 - progress
            else:
                raise ValueError('Unknown factor %s.' % name)
        return jnp.asarray(ret, dtype=jnp.float32)

    return step_fn

def get_batches(rng, train_dataset, batch_size):
  train_ds_size = len(train_dataset)
  steps_per_epoch = train_ds_size // batch_size
  perms = jax.random.permutation(rng, train_ds_size)
  perms = perms[:steps_per_epoch * batch_size]  # skip incomplete batch
  perms = perms.reshape((steps_per_epoch, batch_size))
  for perm in perms:
    batch = train_dataset[perm]
    batch = {k: jnp.array(v) for k, v in batch.items() }
    yield shard(batch)

def main():
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, FlaxTrainingArguments))
    model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    if (
        os.path.exists(training_args.output_dir)
        and os.listdir(training_args.output_dir)
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
        raise ValueError(
            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
        )

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)

    # Log on each process the small summary:
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
    )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        hf_logging.set_verbosity_info()
        hf_logging.enable_explicit_format()
        hf_logging.enable_default_handler()
    logger.info(f"Training/evaluation parameters {training_args}")

    # Set seed before initializing model.
    set_seed(training_args.seed)

    # Load pretrained model and tokenizer
    config = BertConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
        num_labels=glue_tasks[data_args.task_name].num_labels,
        finetuning_task=data_args.task_name,
        cache_dir=model_args.cache_dir,
    )
    tokenizer = BertTokenizerFast.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
    )
    model = FlaxBertForSequenceClassification.from_pretrained(
        model_args.model_name_or_path,
        config=config,
        cache_dir=model_args.cache_dir,
        seed=training_args.seed,
    )

    is_regression = glue_tasks[data_args.task_name].is_regression
    num_labels = glue_tasks[data_args.task_name].num_labels

    get_split = partial(get_dataset, task_name=data_args.task_name, cache_dir=model_args.cache_dir,
                        tokenizer=tokenizer, max_seq_length=data_args.max_seq_length)

    train_dataset = get_split(split="train") if training_args.do_train else None
    eval_dataset = get_split(split="validation") if training_args.do_eval else None
    test_dataset = get_split(split="test") if training_args.do_predict else None

    optimizer = create_optimizer(model.params, training_args.learning_rate)
    optimizer = jax_utils.replicate(optimizer)

    train_ds_size = len(train_dataset)
    train_batch_size = training_args.train_batch_size
    num_epochs = int(training_args.num_train_epochs)
    steps_per_epoch = train_ds_size // train_batch_size
    num_train_steps = steps_per_epoch * num_epochs

    learning_rate_fn = create_learning_rate_scheduler(
        factors='constant * linear_warmup * linear_decay',
        base_learning_rate=training_args.learning_rate,
        warmup_steps=max(training_args.warmup_steps, 1),
        steps_per_cycle=num_train_steps - training_args.warmup_steps,
    )

    rng = jax.random.PRNGKey(training_args.seed)
    dropout_rngs = jax.random.split(rng, jax.local_device_count())
    p_train_step = jax.pmap(
            partial(
                train_step,
                apply_fn=model.__call__,
                glue_task=glue_tasks[data_args.task_name],
                lr_scheduler_fn=learning_rate_fn),
            axis_name="batch",
            donate_argnums=(0,))

    if training_args.do_train:
        i = 0
        for epoch in range(1, num_epochs + 1):
            rng, input_rng = jax.random.split(rng)
            for batch in get_batches(input_rng, train_dataset, train_batch_size):
                optimizer, metrics, dropout_rngs = p_train_step(optimizer, batch, dropout_rngs)
                # logging.info('metrics: %s', metrics)
                if i % 10 == 0:
                  print(f"step {i}: {metrics}")
                i += 1

    if training_args.do_eval:
        for batch in get_batches(input_rng, train_dataset, train_batch_size):
                optimizer, metrics, dropout_rngs = p_train_step(optimizer, batch, dropout_rngs)
                # logging.info('metrics: %s', metrics)
                if i % 10 == 0:
                  print(f"step {i}: {metrics}")
                i += 1

if __name__ == "__main__":
    main()


import numpy as np
import cv2
import sys
import argparse
from skimage import morphology, measure
from matplotlib import pyplot as plt

class Transformations():
	"""docstring for Transformations"""
	def __init__(self):
		pass

	""" interval func converter : 0-255 -> 120->180 """
	@staticmethod
	def toBinary(image):
		gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
		_, binaryImage = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV*cv2.THRESH_OTSU)

		return binaryImage
	@staticmethod
	def getBackground(image):
		kernel = np.ones((3,3),np.uint8)
		m_ = cv2.morphologyEx(image,cv2.MORPH_OPEN,kernel, iterations = 2)

		# sure background area
		backgroundImg = cv2.dilate(m_,kernel,iterations=3)

		return backgroundImg

	@staticmethod
	def getForeground(image,size_dist):
		dist_transform = cv2.distanceTransform(image,cv2.DIST_L2,size_dist)
		ret, foregroundImage = cv2.threshold(dist_transform,0.2*dist_transform.max(),255,0)

		return foregroundImage

def main():

	parser = argparse.ArgumentParser(description='Parser')

	parser.add_argument('--input', help='input image')
	#parser.add_argument('--output', help='output image')

	args = parser.parse_args()

	# checking
	assert (args.input.endswith('.png') or args.input.endswith('.jpg') )
	#assert (args.output.endswith('.png') or args.output.endswith('.jpg') )

	# reading input image
	img = cv2.imread(args.input)

	gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
	ret, thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)

	labels = morphology.label(thresh)
	properties = measure.regionprops(labels)

	# noise removal
	kernel = np.ones((3,3),np.uint8)
	opening = cv2.morphologyEx(thresh,cv2.MORPH_OPEN,kernel, iterations = 2)

	# sure background area
	sure_bg = cv2.dilate(opening,kernel,iterations=3)

	# Finding sure foreground area
	dist_transform = cv2.distanceTransform(opening,cv2.DIST_L2,5)
	ret, sure_fg = cv2.threshold(dist_transform,0.2*dist_transform.max(),255,0)

	# Finding unknown region
	sure_fg = np.uint8(sure_fg)
	unknown = cv2.subtract(sure_bg,sure_fg)

	# Marker labelling
	ret, markers = cv2.connectedComponents(sure_fg)

	# Add one to all labels so that sure background is not 0, but 1
	markers = markers+1

	# Now, mark the region of unknown with zero
	markers[unknown==255] = 0

	markers = cv2.watershed(img,markers)
	img[markers == -1] = [0,0,255]

	coin_areas = [float(prop.area) for prop in properties]
	coin_centroids = [prop.centroid for prop in properties]
	coin_perimeter = [prop.perimeter for prop in properties]

	font = cv2.FONT_HERSHEY_SIMPLEX
	for txt in range(len(coin_areas)):
		if txt == 0:
			continue
		fontScale = (( float(coin_areas[txt])-np.min(coin_areas))/(np.max(coin_areas)-np.min(coin_areas)))+0.21
		print ("região : {}       perimetro : {}     area: {}".format(txt,float(coin_perimeter[txt]),float(coin_areas[txt])))

		cv2.putText(img,str(txt),(int(round(coin_centroids[txt][1])),int(round(coin_centroids[txt][0]))), font, fontScale,(255,255,255),1,cv2.LINE_AA)

	#cv2.imshow('input', img)
	#cv2.waitKey(0)

if __name__ == '__main__':
	main()