import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import multiprocessing

import mne
import xml.etree.ElementTree as ET
from braindecode.preprocessing.preprocess import preprocess, Preprocessor, zscore
from braindecode.preprocessing.windowers import create_windows_from_events
from braindecode.datasets import BaseDataset, BaseConcatDataset

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=UserWarning)
mne.set_log_level(verbose='WARNING')

SHHS_PATH = '/scratch/shhs/edfs/shhs1'
SHHS_EVENTS_PATH = '/scratch/shhs/annotations-events-profusion/shhs1'
SELECTED_SUBJECTS_PATH = './preprocess/shhs/selected_shhs1.txt'
SHHS_SAVE_PATH = os.path.join(os.path.split(os.path.split(SHHS_PATH)[0])[0], 'subjects_data')
NUM_CORES = multiprocessing.cpu_count()

if not os.path.exists(SHHS_SAVE_PATH):
    os.makedirs(SHHS_SAVE_PATH, exist_ok=True)

window_size = 30
sfreq = 100
window_size_samples = window_size*sfreq
subject_ids = pd.read_csv(SELECTED_SUBJECTS_PATH, header=None)
subject_ids = subject_ids[0].values.tolist()
raw_paths = [os.path.join(SHHS_PATH, f'{f}.edf') for f in subject_ids]
ann_paths = [os.path.join(SHHS_EVENTS_PATH, f'{f}-profusion.xml') for f in subject_ids]

label_mapping = {
    "Sleep stage W": 0,
    "Sleep stage N1": 1,
    "Sleep stage N2": 1,
    "Sleep stage N3": 2,
    "Sleep stage R": 3,
}
channel_mapping = {
    'eeg': ['EEG'],
    'ecg': ['ECG'],
    'eog': ['EOG(L)', 'EOG(R)'],
    'emg': ['EMG'],
}

class SHHSSleepStaging(BaseConcatDataset):

    def __init__(
        self,
        raw_path=None,
        ann_path=None,
        channels=None,
        preload=False,
        crop_wake_mins=0,
        crop=None,
    ):
        if (raw_path is None) or (ann_path is None) or (channels is None):
            raise Exception("Please provide paths for raw and annotations file!")

        raw, desc = self._load_raw(
            raw_path,
            ann_path,
            channels,
            preload=preload,
            crop_wake_mins=crop_wake_mins,
            crop=crop
        )
        base_ds = BaseDataset(raw, desc)
        super().__init__([base_ds])

    @staticmethod
    def read_annotations(ann_fname):
        labels = []
        t = ET.parse(ann_fname)
        r = t.getroot()

        for i in range(len(r[4])):
            lbl = int(r[4][i].text)
            if lbl == 0:
                labels.append("Sleep stage W")
            elif lbl == 1:
                labels.append("Sleep stage N1")
            elif lbl == 2:
                labels.append("Sleep stage N2")
            elif (lbl == 3) or (lbl == 4):
                labels.append("Sleep stage N3")
            elif lbl == 5:
                labels.append("Sleep stage R")
            else:
                print( "============================== Faulty file =============================")

        labels = np.asarray(labels)
        onsets = [window_size*i for i in range(len(labels))]
        onsets = np.asarray(onsets)
        durations = np.repeat(window_size, len(labels))
        annots = mne.Annotations(onsets, durations, labels)
        return annots

    def _load_raw(
        self,
        raw_fname,
        ann_fname,
        channels,
        preload,
        crop_wake_mins,
        crop,
    ):
        raw = mne.io.read_raw_edf(raw_fname, preload=preload, include=channels)
        annots = self.read_annotations(ann_fname)
        raw.set_annotations(annots, emit_warning=False)
        raw.resample(sfreq, npad="auto")

        if crop_wake_mins > 0:
            # Find first and last sleep stages
            mask = [x[-1] in ["1", "2", "3", "R"] for x in annots.description]
            sleep_event_inds = np.where(mask)[0]

            # Crop raw
            tmin = annots[int(sleep_event_inds[0])]["onset"] - crop_wake_mins * 60
            tmax = annots[int(sleep_event_inds[-1])]["onset"] + crop_wake_mins * 60
            raw.crop(tmin=max(tmin, raw.times[0]), tmax=min(tmax, raw.times[-1]))

        if crop is not None:
            raw.crop(*crop)

        raw_basename = os.path.basename(raw_fname)
        subj_nb = int(raw_basename[-10:-4])
        desc = pd.Series({"subject_id": subj_nb,}, name="")
        return raw, desc

def __get_epochs(windows_subject):
    epochs_data = []
    for epoch in windows_subject.windows:
        epochs_data.append(epoch)
    epochs_data = np.stack(epochs_data, axis=0) # Shape of (num_epochs, num_channels, num_sample_points)
    return epochs_data

def __get_channels(raw, ann):
    channels_data = dict()
    for ch in channel_mapping.keys():
        subject_dataset = SHHSSleepStaging(raw_path=raw, ann_path=ann, channels=channel_mapping[ch], preload=True)
        preprocess(subject_dataset, [Preprocessor(lambda x: x * 1e6)])

        windows_subject_dataset = create_windows_from_events(
                                subject_dataset,
                                window_size_samples=window_size_samples,
                                window_stride_samples=window_size_samples,
                                preload=True,
                                mapping=label_mapping,
                            )
        preprocess(windows_subject_dataset, [Preprocessor(zscore)])

        channels_data[ch] = __get_epochs(*windows_subject_dataset.datasets)
    channels_data['y'] = windows_subject_dataset.datasets[0].y
    channels_data['subject_id'] = windows_subject_dataset.datasets[0].description['subject_id']
    channels_data['epoch_length'] = len(windows_subject_dataset.datasets[0])
    return channels_data, windows_subject_dataset.datasets[0].description['subject_id']

def preprocess_dataset(raw_paths, ann_paths, k, N):
    raw_paths_core = [f for i, f in enumerate(raw_paths) if i%N==k]
    ann_paths_core = [f for i, f in enumerate(ann_paths) if i%N==k]
    for raw, ann in tqdm(zip(raw_paths_core, ann_paths_core), desc="SHHS dataset preprocessing ...", total=len(raw_paths_core)):
        channels_data, subject_num = __get_channels(raw, ann)
        subjects_save_path = os.path.join(SHHS_SAVE_PATH, f"{subject_num}.npz")
        np.savez(subjects_save_path, **channels_data)

p_list = []
for k in range(NUM_CORES):
    process = multiprocessing.Process(target=preprocess_dataset, args=(raw_paths, ann_paths, k, NUM_CORES))
    process.start()
    p_list.append(process)
for i in p_list:
    i.join()


"""
This file defines actions, i.e. functions the URLs are mapped into
The @action(path) decorator exposed the function at URL:

    http://127.0.0.1:8000/{app_name}/{path}

If app_name == '_default' then simply

    http://127.0.0.1:8000/{path}

If path == 'index' it can be omitted:

    http://127.0.0.1:8000/

The path follows the bottlepy syntax.

@action.uses('generic.html')  indicates that the action uses the generic.html template
@action.uses(session)         indicates that the action uses the session
@action.uses(db)              indicates that the action uses the db
@action.uses(T)               indicates that the action uses the i18n & pluralization
@action.uses(auth.user)       indicates that the action requires a logged in user
@action.uses(auth)            indicates that the action requires the auth object

session, db, T, auth, and tempates are examples of Fixtures.
Warning: Fixtures MUST be declared with @action.uses({fixtures}) else your app will result in undefined behavior
"""

import uuid

from py4web import action, request, abort, redirect, URL, Field
from py4web.utils.form import Form, FormStyleBulma
from py4web.utils.url_signer import URLSigner

from yatl.helpers import A
from . common import db, session, T, cache, auth, signed_url
from . models import get_user_email

url_signer = URLSigner(session)

# The auth.user below forces login.
@action('index')
@action.uses('index.html', url_signer, auth.user)
def index():
    user = auth.get_user()
    if user is None:
        redirect(URL('auth/login'))
    if auth.current_user.get('first_name') is not None:
        username = auth.current_user.get('first_name') + " " + auth.current_user.get("last_name")
    else:
        username = None
        redirect(URL('auth/login'))
    return dict(
        # This is an example of a signed URL for the callback.
        get_posts_url = URL('get_posts', signer=url_signer),
        # Add other callbacks here.
        add_post_url = URL('add_post', signer=url_signer),
        delete_post_url= URL('delete_post', signer=url_signer),
        user=user,
        user_email = get_user_email(),
        username =  username,
        get_rating_url = URL('get_rating'),
        set_rating_url = URL('set_rating', signer=url_signer),
        get_likers_url = URL('get_likers')
    )

@action('get_posts')
@action.uses(url_signer.verify(), auth.user)
def get_posts():
    # Complete.
    posts = db(db.post).select().as_list()
    for post in posts:
        r = db(db.auth_user.email == post['user_email']).select().first()
        name = r.first_name + " " + r.last_name if r is not None else "Unknown"
        post["post_author"] = name
    posts.reverse()
    return dict(posts=posts)

@action('add_post', method="POST")
@action.uses(url_signer.verify(), auth.user, db)
def add_post():
    # print('Something happened')
    id = db.post.insert(
        post_text = request.json.get('post_text')
    )
    return dict(id=id) # You need to fill this in.

@action('delete_post', method="POST")
@action.uses(url_signer.verify(), db)
def delete_post():
    id = request.json.get('id')
    if id is not None:
        db(db.post.id == id).delete()
    return "yayerz"

@action('get_rating')
@action.uses(db, auth.user)
def get_rating():
    post_id = request.params.get('post_id')
    user_email = get_user_email()
    assert post_id is not None
    rating_entry = db((db.thumb.post_id == post_id) & (db.thumb.user_email == user_email)).select().first()
    rating = rating_entry.rating if rating_entry is not None else 0
    return dict(rating=rating)

@action('set_rating', method='POST')
@action.uses(url_signer.verify(), db, auth.user)
def set_rating():
    post_id = request.json.get('post_id')
    user_email = get_user_email()
    rating = request.json.get('rating')
    assert post_id is not None and rating is not None
    db.thumb.update_or_insert(
        ((db.thumb.post_id == post_id) &  (db.thumb.user_email == user_email)),
        post_id=post_id,
        user_email=user_email,
        rating=rating
    )
    return "horry shet"

@action('get_likers')
@action.uses(db, auth.user)
def get_likers():
    post_id = request.params.get('post_id')
    assert post_id is not None
    rating_check = request.params.get('rating')
    rating_check = int(rating_check)
    #get likers first
    likers = db((db.thumb.post_id == post_id) & (db.thumb.rating == 2)).select().as_list()
    if len(likers) == 0:
        theString = ""
    else:
        theString = "Liked by "
        for liker in likers:
            r = db(db.auth_user.email == liker['user_email']).select().first()
            name = r.first_name + " " + r.last_name if r is not None else "Unknown"
            theString = theString + str(name)
            theString = theString + ", "
        theString = theString[:-2]
        theString = theString + ". "
    dislikers = db((db.thumb.post_id == post_id) & (db.thumb.rating == 1)).select().as_list()
    if len(dislikers) > 0:
        theString = theString + "Disliked by "
        for liker in dislikers:
            r = db(db.auth_user.email == liker['user_email']).select().first()
            name = r.first_name + " " + r.last_name if r is not None else "Unknown"
            theString = theString + str(name)
            theString = theString + ", "
        theString = theString[:-2]
        theString = theString + "."
        # if rating_check == 2:
        #     theString = "Liked by "
        # else:
        #     theString = "Disliked by "
        # for liker in likers:
        #     r = db(db.auth_user.email == liker['user_email']).select().first()
        #     name = r.first_name + " " + r.last_name if r is not None else "Unknown"
        #     theString = theString + str(name)
        #     theString = theString + ", "
        # theString = theString[:-2]
    return dict(theString=theString)
# Complete.


from base_classes import TensorflowProjectorVisualizer

class EventEmbeddingProjectorVisualizer(TensorflowProjectorVisualizer):
    def __init__(self, embedding_model, dataset):
        super().__init__(embedding_model)
        self.dataset = dataset

    def event_embedding_layer_weights(self):
        from tensorflow.keras.layers import Embedding
        for layer in self.embedding_model.layers:
            if type(layer) == Embedding:
                return layer.weights[0].numpy()

    def event_type(self, event_id):
        if event_id == 0:
            return "padding"
        elif event_id in self.dataset.negative_events:
            return "normal_only"
        else:
            return "normal_anomaly"

    def write_tsv(self, v_tsv, m_tsv):
        m_tsv.writerow(["event_id", "event_type"])
        for e, v in enumerate(self.event_embedding_layer_weights()):
            m_tsv.writerow([str(e), self.event_type(e)])
            v_tsv.writerow(v)

class SequenceEmbeddingProjectorVisualizer(TensorflowProjectorVisualizer):
    def __init__(self, embedding_model, dataset):
        super().__init__(embedding_model)
        self.dataset = dataset

    def tsv_iter(self, x):
        return zip(x, self.embed(x))

    def write_tsv(self, v_tsv, m_tsv):
        m_tsv.writerow(["sequence", "set", "label", "set_label"])

        for seq, vec in self.tsv_iter(self.dataset.train_data_table[0]):
            v_tsv.writerow(vec)
            m_tsv.writerow([str(seq), "train", "non-anomaly", "train_non"])

        for seq, vec in self.tsv_iter(self.dataset.train_data_table[1]):
            v_tsv.writerow(vec)
            m_tsv.writerow([str(seq), "train", "anomaly", "train_ano"])

        for seq, vec in self.tsv_iter(self.dataset.test_data_table[0]):
            v_tsv.writerow(vec)
            m_tsv.writerow([str(seq), "test", "non-anomaly", "test_non"])

        for seq, vec in self.tsv_iter(self.dataset.test_data_table[1]):
            v_tsv.writerow(vec)
            m_tsv.writerow([str(seq), "test", "anomaly", "test_ano"])


import os

DATA_DIRECTORY = "Data" + os.sep

GENEONTOLOGY_FILENAME = "go-basic.obo"
GENE2GO_FILENAME = "gene2go"
GENEINFO_FILE_SUFFIX = ".gene_info"
COSTANZO_FILENAME = "sgadata_costanzo2009_rawdata_101120.txt"

TRAINING_SCORE_COLUMN = "Score"
TRAINING_PVALUE_COLUMN = "PValue"


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

CSVFILE = "100nmSiO2.csv"  # head300nmSiO2.csv = only 10 rows of data
phi_i = 70 * np.pi / 180  # converting incident angle from deg (first number) to rad
d_L = 100  # thickness of layer in nm
n_air = 1  # refractive index of air
rerange = 5  # upper limit for real part
imrange = 1  # upper limit for imaginary part
# i = 88  # only look at one wavelength (row in csv)

csv = np.loadtxt(CSVFILE, delimiter=",", skiprows=1)

lsp_re = np.linspace(1, rerange, 1001)
lsp_im = np.linspace(0.01, imrange, 1001)
# re, im = np.meshgrid (lsp_re, lsp_im, copy=False)
# n_L = 1j * np.round(im,6) + np.round(re,6)
# n_L = n_L.flatten() # create onedimensional array
n_L = lsp_re

def snell(phi, n1, n2):
  """Calculates the refractive angle, parameters are incident angle phi, refractive index of first medium n1 and of second medium n2"""
  phi_ref = np.arcsin((n1/n2)*np.sin(phi))
  return phi_ref

def fresnel(n1, phi1, n2, phi2):
    """Takes refractive indices and angles of two layers to calculate the amplitude reflection coefficients"""
    rs = (n1 * np.cos(phi1) - n2 * np.cos(phi2)) / (n1 * np.cos(phi1) + n2 * np.cos(phi2))
    rp = (n2 * np.cos(phi1) - n1 * np.cos(phi2)) / (n2 * np.cos(phi1) + n1 * np.cos(phi2))
    return rs, rp

def calc_diff(n_L, rho_giv):
    #Snell's Law:
    phi_L = snell(phi_i, n_air, n_L)
    phi_S = snell(phi_L, n_L, n_S)
    # Fresnel equations:
    # air/layer:
    rs_al, rp_al = fresnel(n_air, phi_i, n_L, phi_L)
    # layer/substrate:
    rs_ls, rp_ls = fresnel(n_L, phi_L, n_S, phi_S)

    beta = (2 * np.pi / lambda_vac) * d_L * n_L * np.cos(phi_L)
    rp_L = (rp_al + rp_ls * np.exp(-2 * 1j * beta)) / (
        1 + rp_al * rp_ls * np.exp(-2 * 1j * beta))
    rs_L = (rs_al + rs_ls * np.exp(-2 * 1j * beta)) / (
        1 + rs_al * rs_ls * np.exp(-2 * 1j * beta))
    rho_L = rp_L / rs_L
    return abs(rho_giv - rho_L), rho_L

lambda_vac = csv[i][0]
n_S = (csv[i][3] + 1j * csv[i][4])

# psi is in our csv-file at index 1, delta at index 2 at row "i" for lambda
n_array = []
rho_array = []
for i, row in enumerate(csv):
    lambda_vac = csv[i][0]
    psi = csv[i][1] * (np.pi/180)
    delta = csv[i][2] * (np.pi/180)
    n_S = csv[i][3] + csv[i][4] * 1j
    rho_giv = np.tan(psi) * np.exp(1j * delta)
    diff, rho_L = calc_diff(n_L, rho_giv)
    idx = np.argmin(diff)  # index of the minimum
    minimum = min(diff)
    n_array = np.append(n_array, n_L[idx])
    rho_array = np.append(rho_array, rho_L[idx])
psi_L = np.arctan(abs(rho_array)) * 180/np.pi
delta_L = np.angle(rho_array) * 180/np.pi
# print("the layer has the refractive index n_L = " , n_array)


# How to delete instance variable from the object:
# 1. Within a class we can delete instance variable as follows
# 	del self.variableName
# 2. From outside of class we can delete instance variables as follows
# 	del object reference.variableName

# Ex:1
class Student:
    def __init__(self):
        self.a=10
        self.b=20
        self.c=30
        self.d=40
    def delete(self):
        del self.a
        del self.b
s=Student()
s.delete()
print(s.__dict__)

# Ex:2
class Student:
    def __init__(self):
        self.a=10
        self.b=20
        self.c=30
s=Student()
del s.a
print(s.__dict__)


#! /usr/bin/env python

import os
from astropy.io import fits
from termcolor import colored

def rc(fname, ext=1):
    eop = fname.rfind('/') #eop = End Of Path; "rfind" finds last occurence of designated string

    if eop != -1:
        path = fname[:eop+1]
        file_only = fname[eop+1:]
    else:
        path = os.getcwd() + '/'
        file_only = fname

    print ('')
    print ('Reading in extension '+colored(ext, 'white')+' of FITS file '+colored(file_only, 'white')+' and copying table')
    print ('File located at: '+colored(path, 'white'))
    print ('')

    f   = fits.open(fname)
    fd  = f[ext].data
    fdc = fd.copy()

    return fdc


from tkinter import *
from random import randrange as rnd, choice
import time
root = Tk()
root.geometry('800x800')

canv = Canvas(root,bg='white')
canv.pack(fill=BOTH,expand=1)
colors = ['red','orange','yellow','green','blue']
a = [[0,0,0] for i in range (0,5)]
p=0
vsego=0
velocity = [[0,0] for i in range (0,5)]
clicked = [0,0,0,0,0]

def new_ball():
   clicked = [0,0,0,0,0]
   global now,vsego,velocity,clicked
   canv.delete(ALL)
   now=rnd(1,5)
   vsego=vsego+now
   for i in range (0,5):
	   velocity[i][0]=rnd(-5,5)
	   velocity[i][1]=rnd(-5,5)
	   if (i<now):
		   x = rnd(100,700)
		   y = rnd(100,500)
		   r = rnd(30,50)
		   a[i]=[x,y,r]
	   else:
		   a[i]=[0,0,0]
   root.after(2000,new_ball)

def balls_movement():
	for i in range (0,5):
		x=a[i][0]
		y=a[i][1]
		r=a[i][2]
		canv.create_oval(x-r,y-r,x+r,y+r,fill = 'white', width=0)
		if ((x-r)<0 or (x+r)>800):
			velocity[i][0]*=-1
		if ((y-r)<0 or (y+r)>800):
			velocity[i][1]*=-1

		a[i][0]=a[i][0]+velocity[i][0]
		a[i][1]=a[i][1]+velocity[i][1]
		x=a[i][0]
		y=a[i][1]
		r=a[i][2]
		canv.create_oval(x-r,y-r,x+r,y+r,fill = colors[i], width=0)
	root.after(1,balls_movement)

def click(event):
	global p
	for i in range (0,5):
		r_nov = (event.x - a[i][0])*(event.x - a[i][0]) + (event.y - a[i][1])*(event.y - a[i][1])
		if ((r_nov < a[i][2] * a[i][2]) and (clicked[i]==0)):
			clicked[i]=1
			p = p+1
	print('score: ',p,'  percent: {:.3}'.format(p*100/vsego))
	l['text'] = 'score: '+str(p)+'  percent: {:.3}'.format(p*100/vsego)

l = Label(bg='black', fg='white', width=20)
l['text'] = 'score: '+str(p)
l.pack()
new_ball()
balls_movement()
canv.bind('<Button-1>', click)
mainloop()


#!/usr/bin/env python3
N = int(input())
V = list(map(int, input().split()))
V.sort()

for i in range(len(V)-1):
    avg = (V[0] + V[1]) / 2
    V.pop(0)
    V.pop(0)
    V.insert(0, avg)

print(V[0])

## self AC


import re
from button import Button
from tagreader import TagReader
from player import Player
from display import Display
from os import listdir
from os.path import isdir, join
import RPi.GPIO as GPIO

# Initialize display first to write Hello
print("Initializing Display")
d = Display()
d.draw_text("Hallo!", 32)

library_path = "/srv/library"

GPIO.setmode(GPIO.BCM)

print("Initializing Player")
p = Player(24)

def on_tag_discovered(tag):
     # Check if there is a album with this name
     try:
         selected_album = (join(library_path, f) for f
                           in listdir(library_path)
                           if isdir(join(library_path, f))
                               and bool(re.search(tag + "$", f, re.I))).next()
         p.play_album(selected_album)
     except StopIteration:
         d.draw_text(tag, 12)

p.track_changed += lambda number: d.draw_text(str(number), 40)
p.stopped += lambda: d.clear()

# We have a few buttons
print("Initializing Buttons")
button_prev = Button(17, callback=p.previous)
button_next = Button(27, callback=p.next)
button_pause = Button(18, callback=p.toggle_pause)
button_volup = Button(23, callback=p.volume_up)
button_voldown = Button(22, callback=p.volume_down)

print("Initializing Tag Reader")
tr = TagReader('tty:AMA0:pn532')
tr.tag_discovered += on_tag_discovered

d.clear()
print("Done")

try:
    while True:
        pass
except KeyboardInterrupt:
    pass

GPIO.cleanup()


from seleniumbase import BaseCase
BaseCase.main(__name__, __file__)

class MyTestClass(BaseCase):
    def test_xkcd(self):
        self.open("https://xkcd.com/353/")
        self.assert_title("xkcd: Python")
        self.assert_element('img[alt="Python"]')
        self.click('a[rel="license"]')
        self.assert_text("free to copy and reuse")
        self.go_back()
        self.click_link("About")
        self.assert_exact_text("xkcd.com", "h2")
        self.click_link("comic #249")
        self.assert_element('img[alt*="Chess"]')


import bisect
class Solution(object):
    def searchMatrix(self, matrix, target):
        rowLength = len(matrix)
        if rowLength == 0:
            return False
        colLength = len(matrix[0])
        if colLength == 0:
            return False
        firstCol = self.getFirstCol(matrix)
        candidateRowIdx, foundIt = self.getRowIdx(firstCol, target)
        if candidateRowIdx == -1:
            return False
        if foundIt :
            return True
        i = bisect.bisect_left(matrix[candidateRowIdx], target)
        if i != colLength and matrix[candidateRowIdx][i] == target:
            return True
        else:
            return False
    def getFirstCol(self,matrix):
        firstCol = []
        for i in range(len(matrix)):
            firstCol.append(matrix[i][0])
        return firstCol

    def getRowIdx(self, l, target):
        idx = bisect.bisect_left(l, target)
        if idx < len(l) and l[idx] == target:
            return (idx, True)
        else:
            return (idx - 1, False)


inspect_dir = "/Users/lucky/Desktop/bdd100k/bdd100k_daytime/bdd100k_annotation_train/"
# inspect_dir = "/Users/lucky/Desktop/VOCdevkit/VOC2007/Annotations/"
import xml.etree.ElementTree as ET
import os
for xml in os.listdir(inspect_dir):
    tree = ET.parse(inspect_dir + xml)
    root = tree.getroot()

    for obj in root.findall('object'):
        class_name = obj.find('name').text
        bb = obj.find('bndbox')
        x_min = int(bb[0].text)
        y_min = int(bb[1].text)
        x_max = int(bb[2].text)
        y_max = int(bb[3].text)
        if x_min >= x_max:
            print("Inspecting " + xml)
            print("Trivial X cooridnate problem")
        if y_min >= y_max:
            print("Inspecting " + xml)
            print("Trivial Y cooridnate problem")

        if  x_min < 0 or x_min > 319 or\
            y_min < 0 or y_min > 319 or\
            x_max < 0 or x_max > 319 or\
            y_max < 0 or y_max > 319:
            print("Inspecting " + xml)
            print("Exceed legit boudnary")

        # for i in range(4):
        #     if int(bb[i].text) == 319:
        #         print("GGGGGGGGGGGGGGGGGGG")
        # print(int(bb[0].text))
        # bb[1].text
        # bb[2].text
        # bb[3].text
        # result_dic[img_num].append((class_name, "annotate", bb[0].text, bb[1].text, bb[2].text, bb[3].text))


import sys

directions = {
    'N': (0, 1),
    'S': (0, -1),
    'E': (1, 0),
    'W': (-1, 0)
}

def part1(instructions):
    angle_direction = { 0: 'E', 90: 'N', 180: 'W', 270: 'S' }
    ferryx = ferryy = angle = 0
    for action, value in instructions:
        if action in 'LR':
            value = value if action == 'L' else 360 - value
            angle = (angle + value) % 360
        else:
            dx, dy = directions[angle_direction[angle]] if action == 'F' else directions[action]
            ferryx += value * dx
            ferryy += value * dy
    return abs(ferryx) + abs(ferryy)

def part2(instructions):
    waypointx, waypointy = 1, 10
    ferryx = ferryy = 0
    for action, value in instructions:
        if action in 'LR':
            value = value if action == 'L' else 360 - value
            if value == 180:
                waypointx, waypointy = -waypointx, -waypointy
            elif value == 90:
                waypointx, waypointy = waypointy, -waypointx
            else:
                waypointx, waypointy = -waypointy, waypointx
        elif action == 'F':
            ferryx += value * waypointx
            ferryy += value * waypointy
        else:
            waypointx += value * directions[action][1]
            waypointy += value * directions[action][0]
    return abs(ferryx) + abs(ferryy)

assert len(sys.argv) == 2
instructions = [(instruction[0], int(instruction[1:])) for instruction in open(sys.argv[1]).read().split()]

print(f'Part 1: {part1(instructions)}, Part 2: {part2(instructions)}')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('signups', '0007_auto_20171029_0941'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='userprofile',
            name='field',
        ),
        migrations.AddField(
            model_name='userprofile',
            name='first_name',
            field=models.CharField(max_length=120, null=True, blank=True),
        ),
        migrations.AddField(
            model_name='userprofile',
            name='image',
            field=models.URLField(default=b'https://a3-images.myspacecdn.com/images03/1/240e42b5d9ce48a78983961e7fcb3c39/600x600.jpg', max_length=500, blank=True),
        ),
        migrations.AddField(
            model_name='userprofile',
            name='last_name',
            field=models.CharField(max_length=120, null=True, blank=True),
        ),
        migrations.AlterField(
            model_name='signup',
            name='image',
            field=models.URLField(max_length=500, blank=True),
        ),
    ]


##############################################################
##########	Object Tracking using OpenCV and Python	##########
##############################################################

#############################
#####    Optical Flow	#####
#############################

#####	Importing Libraries		#####

import cv2
import numpy as np

#####	Pre-Processing	#####

cap = cv2.VideoCapture("Videos/chaplin.mp4")										#Video Capture
ret, first_frame = cap.read()														#Get the first frame
prev_gray = cv2.cvtColor(first_frame,cv2.COLOR_BGR2GRAY)							#Conver to Grayscale
mask = np.zeros_like(first_frame)   												#Create a Mask
mask[..., 1] = 255																	#Setting saturation to max

#####	Optical Dense Flow 	#####

while(cap.isOpened()):																#While Loop
	ret, frame = cap.read()															#Get Video
	cv2.imshow('imput', frame)														#Display the output
	gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)									#Conver to Grayscale
	flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)	#Farneback Optical FLow Calculation
	magn, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1])						#Calculate Mag and Angle
	mask[..., 0] = angle*180/np.pi/2												#Set the Optical Flow Direction
	mask[..., 2] = cv2.normalize(magn, None, 0, 255, cv2.NORM_MINMAX)				#Normalise The Magnitude
	rgb = cv2.cvtColor(mask, cv2.COLOR_HSV2RGB)										#Convert the HSB to RGB
	cv2.imshow("Dense Optical Flow", rgb)

#####	UPDATION	#####

	prev_gray = gray

#####	END		#####

	if cv2.waitKey(300) & 0xFF == ord("q"):
            break

cap.release()
cv2.destroyAllWindows()


from sqlalchemy import Column, Date, ForeignKey, Integer, MetaData, String, Table
from sqlalchemy.orm import registry, relationship

from ddd_python.domain import model

metadata = MetaData()

mapper_registry = registry()

order_lines = Table(
    "order_lines",
    metadata,
    Column("id", Integer, primary_key=True, autoincrement=True),
    Column("batch_id", Integer, ForeignKey("batches.id")),
    Column("sku", String(255), nullable=False),
    Column("qty", Integer, nullable=False),
    Column("orderid", String(255)),
)

batches = Table(
    "batches",
    metadata,
    Column("id", Integer, primary_key=True, autoincrement=True),
    Column("product_id", Integer, ForeignKey("products.id")),
    Column("reference", String(255), nullable=False, unique=True),
    Column("qty", Integer, nullable=False),
    Column("eta", Date, nullable=False),
    Column("sku", String(255), nullable=False),
)

products = Table(
    "products",
    metadata,
    Column("id", Integer, primary_key=True, autoincrement=True),
    Column("sku", String(255), nullable=False, unique=True),
    Column("version_number", Integer, nullable=False, default=0),
)

allocations_view = Table(
    "allocations_view",
    metadata,
    Column("orderid", String(255)),
    Column("sku", String(255)),
    Column("batchref", String(255)),
)

products_view = Table(
    "products_view",
    metadata,
    Column("sku", String(255)),
)

batches_view = Table(
    "batches_view",
    metadata,
    Column("sku", String(255)),
    Column("reference", String(255)),
    Column("eta", Date),
    Column("qty", Integer),
)

def start_mappers():
    mapper_registry.map_imperatively(model.OrderLine, order_lines)
    mapper_registry.map_imperatively(
        model.Batch,
        batches,
        properties={
            "_purchased_quantity": batches.c.qty,
            "allocations": relationship(model.OrderLine, backref="batch"),
        },
    ),
    mapper_registry.map_imperatively(
        model.Product,
        products,
        properties={"batches": relationship(model.Batch, backref="product")},
    )

def clear_mappers():
    mapper_registry.dispose()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Project: ShanghaiShenzhenStock
# File name: step1_test_equal_portfolio
# Author: warn
# Date: 2017/4/10

import os

import pandas as pd

from ChineseStock.constants import Constant as const
from ChineseStock.wealth_generator.calculate_wealth_info_equal_portfolio import calculate_raw_alpha_wealth_series_record

portfolio_range = [15, 20, 22, 25, 30]
file_path = '20170410_test_equal_portfolio'
input_report_path = os.path.join(const.TEMP_PATH, '20170409_test', 'input_return_report')
raw_wealth_path = os.path.join(const.TEMP_PATH, file_path, 'raw_wealth')
alpha_wealth_path = os.path.join(const.TEMP_PATH, file_path, 'alpha_wealth')
buy_sell_path = os.path.join(const.TEMP_PATH, file_path, 'buy_sell_record')

for path_name in [raw_wealth_path, alpha_wealth_path, buy_sell_path]:
    if not os.path.isdir(path_name):
        os.makedirs(path_name)

tag = 'test'

def multi_process_wealth_series(portfolio_info):
    hday = portfolio_info['hday']
    sr = portfolio_info['sr']
    p = portfolio_info['p']
    return calculate_raw_alpha_wealth_series_record(hday, sr, input_report_path, raw_wealth_path, alpha_wealth_path,
                                                    buy_sell_path, tag, p)

if __name__ == '__main__':
    import pathos
    import multiprocessing

    from ChineseStock.utilities.calc_util import get_annualized_return, get_max_draw_down, get_sharpe_ratio

    pool = pathos.multiprocessing.ProcessingPool(multiprocessing.cpu_count() - 2)

    import time

    start_time = time.time()
    # calculate_raw_alpha_wealth_series(5, 1, input_report_path, raw_wealth_path, alpha_wealth_path, tag, 15)
    # print(time.time() - start_time)

    # parameter_list = []
    # for hday in [5, 10, 12, 15, 20]:
    #     for sr in [1, 2, 3, 4, 5]:
    #         for p in portfolio_range:
    #             parameter_list.append({'hday': hday, 'sr': sr, 'p': p})
    #
    # pool.map(multi_process_wealth_series, parameter_list)

    file_list = os.listdir(raw_wealth_path)
    df = pd.read_pickle(os.path.join(raw_wealth_path, file_list[0]))

    result_df = pd.DataFrame(index=df.index)

    for f in file_list:
        df = pd.read_pickle(os.path.join(raw_wealth_path, f))
        col_name = f[:-2]
        result_df[col_name] = df

    result_df.to_excel(os.path.join(const.TEMP_PATH, file_path, '20170410_insider_test_result.xlsx'))
    sta_df = pd.DataFrame(columns=result_df.keys())
    sta_df.loc[const.SHARPE_RATIO] = get_sharpe_ratio(result_df, const.WEALTH_DATAFRAME)
    sta_df.loc[const.ANNUALIZED_RETURN] = get_annualized_return(result_df, const.WEALTH_DATAFRAME)
    sta_df.loc['max_drawdown'] = result_df.apply(get_max_draw_down)

    sta_df.to_excel(os.path.join(const.TEMP_PATH, file_path, '20170410_insider_test_statistics.xlsx'))

    print(time.time() - start_time)


import h5py
import numpy as np
import argparse
import sys, os
import subprocess
import csv, yaml
import math

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader

try:
    import horovod.torch as hvd
except:
    hvd = None

nthreads = int(os.popen('nproc').read()) ## nproc takes allowed # of processes. Returns OMP_NUM_THREADS if set
print("NTHREADS=", nthreads, "CPU_COUNT=", os.cpu_count())
torch.set_num_threads(nthreads)

parser = argparse.ArgumentParser()
parser.add_argument('--epoch', action='store', type=int, default=50, help='Number of epochs')
parser.add_argument('--batch', action='store', type=int, default=128, help='Batch size')
parser.add_argument('-o', '--outdir', action='store', type=str, required=True, help='Path to output directory')
parser.add_argument('--lr', action='store', type=float, default=1e-3, help='Learning rate')
parser.add_argument('--batchPerStep', action='store', type=int, default=1, help='Number of batches per step (to emulate all-reduce)')
parser.add_argument('--shuffle', action='store', type=bool, default=True, help='Shuffle batches for each epochs')
parser.add_argument('--optimizer', action='store', choices=('sgd', 'adam', 'radam', 'ranger',
    'lamb', 'novograd'), default='adam', help='optimizer to run')
parser.add_argument('--model', action='store', choices=('default', 'defaultnorm1', 'defaultnorm0', 'defaultcat', 'defaultnorm1cat',
                                                        'log3ch', 'log3chnorm1', 'log3chnorm0', 'log3chcat', 'log3chnorm1cat',
                                                        'log5ch', 'log5chnorm1', 'log5chnorm0', 'log5chcat', 'log5chnorm1cat',
                                                        'original',
                                                        'circpad', 'circpadnorm1', 'circpadnorm0', 'circpadnorm1cat',
                                                        'circpadlog3ch', 'circpadlog3chnorm1', 'circpadlog3chnorm0', 'circpadlog3chnorm1cat',
                                                        'circpadlog5ch', 'circpadlog5chnorm1', 'circpadlog5chnorm0', 'circpadlog5chnorm1cat',
                                                        'CPlargekernel', 'CPlargekernelnorm0', 'CPlargekernelnorm1',),
                               default='default', help='choice of model')
parser.add_argument('--device', action='store', type=int, default=0, help='device name')
parser.add_argument('-c', '--config', action='store', type=str, default='config.yaml', help='Configration file with sample information')
parser.add_argument('--lars','-l', action='store', type=bool, default=False, help='Use LARS optimizer')
parser.add_argument('--noperfmon', action='store_true', default=False, help='turn off performance monitoring')
parser.add_argument('--abs', action='store_true', default=False, help='if true, use absoulute value for weight')

args = parser.parse_args()

config = yaml.load(open(args.config).read(), Loader=yaml.FullLoader)

hvd_rank, hvd_size = 0, 1
if hvd:
    hvd.init()
    hvd_rank = hvd.rank()
    hvd_size = hvd.size()
    print("Horovod is available. (rank=%d size=%d)" % (hvd_rank, hvd_size))
    if torch.cuda.is_available(): torch.cuda.set_device(hvd.local_rank())
if torch.cuda.is_available() and args.device >= 0: torch.cuda.set_device(args.device)

if not os.path.exists(args.outdir): os.makedirs(args.outdir)
modelFile = os.path.join(args.outdir, 'model.pth')
weightFile = os.path.join(args.outdir, 'weight_%d.pth' % hvd_rank)
predFile = os.path.join(args.outdir, 'predict_%d.npy' % hvd_rank)
trainingFile = os.path.join(args.outdir, 'history_%d.csv' % hvd_rank)
resourceByCPFile = os.path.join(args.outdir, 'resourceByCP_%d.csv' % hvd_rank)
resourceByTimeFile = os.path.join(args.outdir, 'resourceByTime_%d.csv' % hvd_rank)

if not args.noperfmon:
    proc = subprocess.Popen(['python', '../scripts/monitor_proc.py', '-t', '1',
                            '-o', resourceByTimeFile, '%d' % os.getpid()],
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

import time
class TimeHistory():
    def on_train_begin(self):
        self.times = []
    def on_epoch_begin(self):
        self.epoch_time_start = time.time()
    def on_epoch_end(self):
        self.times.append(time.time() - self.epoch_time_start)

sys.path.append("../scripts")
from monitor_proc import SysStat
sysstat = SysStat(os.getpid(), fileName=resourceByCPFile)
sysstat.update(annotation="start_loggin")

sys.path.append("../python")
from HEPCNN.dataset_hepcnn import HEPCNNDataset as MyDataset

sysstat.update(annotation="add samples")
myDataset = MyDataset()
for sampleInfo in config['samples']:
    if 'ignore' in sampleInfo and sampleInfo['ignore']: continue
    name = sampleInfo['name']
    myDataset.addSample(name, sampleInfo['path'], weight=sampleInfo['xsec']/sampleInfo['ngen'])
    myDataset.setProcessLabel(name, sampleInfo['label'])
sysstat.update(annotation="init dataset")
myDataset.initialize(logger=sysstat)

sysstat.update(annotation="split dataset")
lengths = [int(0.6*len(myDataset)), int(0.2*len(myDataset))]
lengths.append(len(myDataset)-sum(lengths))
torch.manual_seed(config['training']['randomSeed1'])
trnDataset, valDataset, testDataset = torch.utils.data.random_split(myDataset, lengths)
torch.manual_seed(torch.initial_seed())

kwargs = {'num_workers':min(config['training']['nDataLoaders'], nthreads), 'pin_memory':False}

if hvd:
    trnSampler = torch.utils.data.distributed.DistributedSampler(trnDataset, num_replicas=hvd_size, rank=hvd_rank)
    valSampler = torch.utils.data.distributed.DistributedSampler(valDataset, num_replicas=hvd_size, rank=hvd_rank)
    trnLoader = DataLoader(trnDataset, batch_size=args.batch, sampler=trnSampler, **kwargs)
    valLoader = DataLoader(valDataset, batch_size=args.batch, sampler=valSampler, **kwargs)
else:
    trnLoader = DataLoader(trnDataset, batch_size=args.batch, shuffle=args.shuffle, **kwargs)
    valLoader = DataLoader(valDataset, batch_size=args.batch, shuffle=False, **kwargs)

## Build model
sysstat.update(annotation="Model start")
if args.model == 'original':
    from HEPCNN.model_original import MyModel
elif 'largekernel' in args.model:
    from HEPCNN.model_largekernel import MyModel
elif 'circpad' in args.model:
    from HEPCNN.model_circpad import MyModel
else:
    from HEPCNN.model_default import MyModel
model = MyModel(myDataset.width, myDataset.height, model=args.model)

if hvd_rank == 0: torch.save(model, modelFile)
device = 'cpu'
if args.device >= 0 and torch.cuda.is_available():
    model = model.cuda()
    device = 'cuda'

if args.optimizer == 'radam':
    from optimizers.RAdam import RAdam
    optm = RAdam(model.parameters(), lr=args.lr)
elif args.optimizer == 'ranger':
    from optimizers.RAdam import RAdam
    from optimizers.Lookahead import Lookahead
    optm_base = RAdam(model.parameters(), lr=args.lr)
    optm = Lookahead(optm_base)
elif args.optimizer == 'adam':
    optm = optim.Adam(model.parameters(), lr=args.lr)
elif args.optimizer == 'sgd':
    optm = optim.SGD(model.parameters(), lr=args.lr)
elif args.optimizer == 'lamb':
    from optimizers.lamb import Lamb
    optm = Lamb(model.parameters(), lr=args.lr)
elif args.optimizer == 'novograd':
    from optimizers.novograd import NovoGrad
    optm = NovoGrad(model.parameters(), lr=args.lr)
else:
    print("Cannot find optimizer in the list")
    exit()

if args.lars == True:
    from optimizers.lars import LARS
    optm_base = optm
    optm = LARS(optimizer=optm_base)
    print("Using LARS with base optimizer %s"%(args.optimizer))

if hvd:
    compression = hvd.Compression.none
    optm = hvd.DistributedOptimizer(optm,
                                    named_parameters=model.named_parameters(),
                                    compression=compression, backward_passes_per_step=args.batch)
    hvd.broadcast_parameters(model.state_dict(), root_rank=0)
    hvd.broadcast_optimizer_state(optm, root_rank=0)

def metric_average(val, name):
    tensor = torch.tensor(val)
    avg_tensor = hvd.allreduce(tensor, name=name)
    return avg_tensor.item()

sysstat.update(annotation="modelsetup_done")

with open(args.outdir+'/summary.txt', 'w') as fout:
    fout.write(str(args))
    fout.write('\n\n')
    fout.write(str(model))
    fout.close()

from tqdm import tqdm
from sklearn.metrics import accuracy_score
bestWeight, bestLoss = {}, 1e9
try:
    timeHistory = TimeHistory()
    timeHistory.on_train_begin()
    sysstat.update(annotation="train_start")
    history = {'time':[], 'loss':[], 'acc':[], 'val_loss':[], 'val_acc':[]}
    for epoch in range(args.epoch):
        timeHistory.on_epoch_begin()
        sysstat.update(annotation='epoch_begin')
        model.train()
        trn_loss, trn_acc = 0., 0.
        optm.zero_grad()

        for i, (data, label, _, rescale, procIdx, evtWeight) in enumerate(tqdm(trnLoader, desc='epoch %d/%d' % (epoch+1, args.epoch))):
            data = data.float().to(device)
            label = label.float().to(device)
            rescale = rescale.float().to(device)
            rescaledWeight = evtWeight.float().to(device)*rescale

            if(args.abs) : rescaledWeight = torch.abs(rescaledWeight)

            pred = model(data)
            crit = torch.nn.BCEWithLogitsLoss(weight=rescaledWeight)
            if device == 'cuda': crit = crit.cuda()
            l = crit(pred.view(-1), label)
            l.backward()
            if i % args.batchPerStep == 0 or i+1 == len(trnLoader):
                optm.step()
                optm.zero_grad()

            trn_loss += l.item()
            trn_acc += accuracy_score(label.to('cpu'), np.where(pred.to('cpu') > 0.5, 1, 0), sample_weight=rescaledWeight.to('cpu'))

            sysstat.update()
        trn_loss /= len(trnLoader)
        trn_acc  /= len(trnLoader)
        model.eval()
        val_loss, val_acc = 0., 0.
        for i, (data, label, _, rescale, procIdx, evtWeight) in enumerate(tqdm(valLoader)):
            data = data.float().to(device)
            label = label.float().to(device)
            rescale = rescale.float().to(device)
            rescaledWeight = evtWeight.float().to(device)*rescale

            if(args.abs) : rescaledWeight = torch.abs(rescaledWeight)

            pred = model(data)
            crit = torch.nn.BCEWithLogitsLoss(weight=rescaledWeight)
            loss = crit(pred.view(-1), label)

            val_loss += loss.item()
            val_acc += accuracy_score(label.to('cpu'), np.where(pred.to('cpu') > 0.5, 1, 0), sample_weight=rescaledWeight.to('cpu'))
        val_loss /= len(valLoader)
        val_acc  /= len(valLoader)

        if hvd: val_acc = metric_average(val_acc, 'avg_accuracy')
        if bestLoss > val_loss:
            bestWeight = model.to('cpu').state_dict()
            bestLoss = val_loss
            if hvd_rank == 0:
                torch.save(bestWeight, weightFile)
                sysstat.update(annotation="saved_model")
            model.to(device)

        timeHistory.on_epoch_end()
        sysstat.update(annotation='epoch_end')
        history['loss'].append(trn_loss)
        history['acc'].append(trn_acc)
        history['val_loss'].append(val_loss)
        history['val_acc'].append(val_acc)

        history['time'].append(timeHistory.times[-1])
        if hvd_rank == 0:
            with open(trainingFile, 'w') as f:
                writer = csv.writer(f)
                keys = history.keys()
                writer.writerow(keys)
                for row in zip(*[history[key] for key in keys]):
                    writer.writerow(row)
            sysstat.update(annotation="wrote_logs")

    sysstat.update(annotation="train_end")

except KeyboardInterrupt:
    print("Training finished early")


def solution(begin, target, words):
    answer = 0
    results = [[begin for i in words]]

    if target not in words:
        return answer

    while True:
        answer += 1
        results.append([])

        for result in results[-2]:
            for word in words:
                if check(result, word):
                    if word==target:
                        return answer
                    else:
                        results[-1].append(word)

    return answer

def check(word1, word2):
    flag = False

    for idx in range(len(word1)):
        if word1[idx] != word2[idx]:
            if flag:
                return False
            else:
                flag = True

    return flag


"""Importing dependencies"""
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import pandas as pd

class PCA():
    """
    This class performs Principle Component Analysis (PCA) on scaled data"""

    def __init__(self,X):
        """
        Initialaze
        """

        self.X = X
        self.covariance_matrix = self.cov()
        self.vals = self.eig()[0]
        self.vecs = self.eig()[1]
        self.projections = self.get_projections()

    def get_vals(self):
        """
        Returns:
                Eigenvalues
        """
        return self.vals

    def get_vecs(self):
        """
        Returns:
                Eigenvectors
        """
        return self.vecs

    def cov(self):
        """
        arr: self.X

        Returns:
                Covariance matrix
        """
        self.covariance_matrix = np.dot(self.X.T, self.X) / (self.X.shape[0] - 1)
        return self.covariance_matrix

    def eig(self):
        """
        arr:
                self.covariance_matrix

        returns:
                sorted eigenvalues highest to lowest
                sortrd eigenvectors by eigenvalues
        """
        self.vals, self.vecs = np.linalg.eig(self.covariance_matrix)

        # Sort eigenvectors and values by eigenvalues
        order = np.argsort(self.vals)[::-1]
        self.vals =self. vals[order]
        self.vecs = self.vecs[:, order]

        return self.vals, self.vecs

    def scree_plot(self):
        """
        arr:
                self.vals
                self.vecs

        returns:
                scree plot
        """
        # Scree plot  -> Kaiser rule:
        plt.bar(np.arange(len(self.vals)) + 1, self.vals, label = 'PC')

        plt.axhline(1, label = 'eigenvalues > 1')
        plt.title("Kaiser rule")
        plt.ylabel('Eigenvalues')
        plt.xlabel('No. of components')
        plt.legend()
        plt.show()

    def get_projections(self):
        """
        arr:
                self.X
                self.vecs
                self.vals

        returns:
               self.projections as pandas dataframe
        """

        pca_dict= {}
        for n_vec in range(len(self.vecs)):

            pca_dict['PC{}'.format(n_vec + 1)] = np.dot(self.X,self.vecs[:, n_vec])
            self.projections = pd.DataFrame(pca_dict)

        return self.projections

    def plot_PCA(self, x = 'PC1', y = 'PC2', color = 'PC3'):
        """
        arr:
                self.projections
        returns:
                PCA plot
        """

        fig = px.scatter(self.projections, x = x, y=y, color = color)

        fig.update_layout(
            width = 600,
            height = 600,
            title = "Priciple Component Analysis"
        )
        fig.show()

def main():
    """
    main function
    """
    print('PCA module')

if __name__ == "__main__":
    main()


import pandas as pd

mymoviedata=pd.read_table("http://bit.ly/movieusers",sep="|",header=None)
print(mymoviedata.head())
mymoviedata.info()

mymoviedata.columns=['sr.no', 'age', 'Gender', 'profession', 'Views']
print(mymoviedata.head())

print(mymoviedata['Gender'])

mymoviedata['agegen']=mymoviedata['age'].astype('str')+":"+mymoviedata['Gender']
print(mymoviedata.head())

mymoviedata=mymoviedata.replace({'Views':'[A-Za-z]'},0,regex=True)
mymoviedata['Views']=mymoviedata['Views'].astype('int64')

g=mymoviedata.groupby(by='age')

g1=g['Views'].mean()
import matplotlib.pyplot as plt
plt.bar(g1.index,g1.values)
plt.xlabel("Age")
plt.ylabel("Average Views")
plt.show()

g=mymoviedata.groupby(by="profession")
g1=g['Views'].mean()
plt.bar(g1.index,g1.values)
plt.xticks(rotation=90)
plt.xlabel("Profession")
plt.ylabel("Total Views")
plt.show()


# Level 1
# 1. Iterate 0 to 10 using for loop, do the same using while loop.
# First solution of "For" loop:
k = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
for i in k:
    print(i)
                                # End.

# Second solution of "For" loop:
for i in range(11):
    print(i)
                                # End.

# First solution of "While" loop:
number = 0
while number < 11:
    print(number)
    number = number + 1
                                # End.

# 2. Iterate 10 to 0 using for loop, do the same using while loop.
# First solution of "For" loop:
k = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
for i in k:
    print(i)
                                # End.

# Second solution of "For" loop:
n = 0
for n in reversed(range(11)):
    print(n)
                                # End.

# First solution of "While" loop:
n = 11
while n >= 0:
    print(n)
    n = n - 1
                                # End.

# Second solution of "While" loop:
n = 0
while n <= 10:
    print(10 - n)
    n = n + 1
                                # End.

# 3. Write a loop that makes seven calls to print(), so we get on the output the following triangle:
'''
  #
  ##
  ###
  ####
  #####
  ######
  ########
'''
# First solution:
n = 0
while n < 7:
    print("#" * (n + 1))
    n = n + 1
                                # End.

# Second solution:
for i in range(0, 7):
    for j in range(0, 1 + i):
        print('#', end = '')
    print()
                                # End.

# Third solution:
for i in range (0, 7):
    for j in range (0, 7):
        if j > i:
            print()
            break
        else:
            print("#", end = "")
                                # End.

# Fourth solution:
sum_line = 7
line = 0
sign = 0
while line < sum_line:
    if (sign) >= (line + 1):
        print()
        line = line + 1
        sign = 0
        continue
    print("*", end="")
    sign = sign + 1
                                # End.

# 4. Use nested loops to create the following:
'''
# # # # # # # #
# # # # # # # #
# # # # # # # #
# # # # # # # #
# # # # # # # #
# # # # # # # #
# # # # # # # #
# # # # # # # #
'''

# First solution:
for i in range(0, 8):
    for j in range(8):
        print('#', end = ' ')
    print()
                                # End.

# 5. Print the following pattern:
'''
0 x 0 = 0
1 x 1 = 1
2 x 2 = 4
3 x 3 = 9
4 x 4 = 16
5 x 5 = 25
6 x 6 = 36
7 x 7 = 49
8 x 8 = 64
9 x 9 = 81
10 x 10 = 100
'''

# 6. Iterate through the list, ['Python', 'Numpy','Pandas','Django', 'Flask'] using a for loop and print out the items.

# 7. Use for loop to iterate from 0 to 100 and print only even numbers

# 8. Use for loop to iterate from 0 to 100 and print only odd numbers

# Level 2
# 1. Use for loop to iterate from 0 to 100 and print the sum of all numbers.
'''The sum of all numbers is 5050.'''

# 2. Use for loop to iterate from 0 to 100 and print the sum of all evens and the sum of all odds.
'''The sum of all evens is 2550. And the sum of all odds is 2500.'''

# Level 3
# 1. Go to the data folder and use the countries.py file. Loop through the countries and extract all the countries containing the word land.

# 2. This is a fruit list, ['banana', 'orange', 'mango', 'lemon'] reverse the order using loop.

# 3. Go to the data folder and use the countries_data.py file.
'''
What are the total number of languages in the data
Find the ten most spoken languages from the data
Find the 10 most populated countries in the world
'''


# FUSE imports
from __future__ import print_function, absolute_import, division

import logging

from collections import defaultdict
from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG
from time import time
import os

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

from cmd import *
from consts import *
from opcodes import *
import packets
import socks

import argparse
import json
import selectors
import signal
import socket
import sys
import threading
from memory import Memory

bootstrap_ip = "127.0.0.1"
bootstrap_sock = None
listen_sock = None
tier = "basic"
mount = "mnt"
filesystem = None
args = None
t = None
fuse = None
sel = selectors.DefaultSelector()
def intro():
    print("***DiFUSE Client***")
    print("Connected to bootstrap " + bootstrap_ip + " port " + str(args.port))
    print("Type help for a list of commands")

def setup_args():
    global args
    parser = argparse.ArgumentParser()
    parser.add_argument("-ip", help="ip of bootstrap")
    parser.add_argument("-p", "--port", nargs='?', default=1234, type=int, help="port of bootstrap")
    parser.add_argument("-m", "--mount", nargs="?", default="mnt", help="FUSE mount dir")
    parser.add_argument("-t", "--tier", choices=["basic", "hash", "repl"], help="DiFUSE hw tier")
    args = parser.parse_args()

# join the network
# global listen_sock is where this node will accept connections from other nodes (incl. bootstrap)
# temp_sock used to send bootstrap join request with addr of listen_sock, then closed
def join():
    global listen_sock
    listen_sock = socks.listen_sock()
    listen_sock.setblocking(False)
    temp_sock = socks.tcp_sock() # only a temp sock is required to join network
    try:
        temp_sock.connect((bootstrap_ip, args.port))
    except:
        print("Could not connect to " + str(bootstrap_ip) + " on port " + str(args.port))
        sys.exit(0)
    print('connected to bootstrap', temp_sock)
    packet = packets.join_packet((socket.gethostbyname(socket.gethostname()),listen_sock.getsockname()[1]))
    print('Sending join request', packet)
    temp_sock.send(packets.build(packet))
    #TODO: recv ack?
    temp_sock.close()

def leave():
    global listen_sock
    temp_sock = socks.tcp_sock()
    temp_sock.connect((bootstrap_ip,args.port))
    print('connected to bootstrap for exit', temp_sock)
    packet = packets.leave_packet(listen_sock.getsockname())
    print('sending leave notice', packet)
    temp_sock.send(packets.build(packet))
    #TODO: recv ack?
    temp_sock.close()

def prompt():
    print("DiFUSE Client> ", end='', flush=True)

def help_menu():
    print("help\t show this menu")
    print("bootstrap_ip\t print bootstrap ip")
    print("args.port\t print bootstrap port")

def invalid():
    print("invalid command")

def accept(sock):
    conn, addr = sock.accept()
    print('accepted connection', conn)
    sel.register(conn, selectors.EVENT_READ, recv)

def close(sock):
    sel.unregister(sock)
    sock.close()

def recv(sock):
    print("receiving from socket")
    packet = packets.unpack(sock.recv(MAX_READ))
    print("packet is: ", packet)
    if not packet:
        print('Read empty packet, closing connection', sock)
        close(sock)
        return
    if packet['opcode'] == OP_GETATTR:
        print('Received OP_GETATTR in node.py')
        reply = packets.new_packet(OP_GETATTR_R)
        reply['getattr'] = filesystem.getattr(packet['path']) # call getattr locally
        print('Sending OP_GETATTR_R', reply)
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_BYE_R:
        close(sock)
        os.kill(os.getpid(),signal.SIGKILL)
        sys.exit(0)

    elif packet['opcode'] == OP_READ:
        print("Received OP_READ in node.py")
        reply = packets.new_packet(OP_READ_R)
        reply['read'] = filesystem.read(packet['path'],packet['size'],packet['offset'],packet['fh']).decode()
        print('Sending OP_READ_R with ', reply['read'])
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_WRITE:
        print("Received OP_WRITE in node.py")
        reply = packets.new_packet(OP_WRITE_R)
        reply['datalen'] = filesystem.write(packet['path'],packet['data'].encode(),packet['offset'],packet['fh'])
        print('Sending OP_WRITE_R with ', reply['datalen'])
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_TRUNC:
        print("Received OP_TRUNC in node.py")
        reply = packets.new_packet(OP_TRUNC_R)
        reply['trunc'] = filesystem.truncate(packet['path'],packet['length'])
        print('Sending OP_TRUNC_R with', reply['trunc'])
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_GETXATTR:
        print("Received OP_GETXATTR in nodde.py")
        reply = packets.new_packet(OP_GETXATTR_R)
        reply['getxattr'] = filesystem.getxattr(packet['path'],packet['name'])
        print('Sending OP_GETXATTR_R with', reply['getxattr'])
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_LEAVE:
        print("Bootstrap has exited, so we will exit too.")
        os.kill(os.getpid(),signal.SIGKILL)
        sys.exit(0)

    elif packet['opcode'] == OP_RENAME:
        print("Received OP_RENAME in node.py")
        reply = packets.new_packet(OP_RENAME_R)
        reply['rename'] = filesystem.rename(packet['old'],packet['new'])
        print("Sending OP_RENAME_R in node.py")
        sock.send(packets.build(reply))

    elif packet['opcode'] == OP_DELETE:
        print("Received OP_DELETE in node.py")
        reply = packets.new_packet(OP_DELETE_R)
        reply['delete'] = filesystem.unlink(packet['path'])
        print("Sending OP_DELETE_R in node.py")
        sock.send(packets.build(reply))
    else:
        print('Invalid opcode')

def do_cmd(stdin):
    cmd = stdin.readline().strip()
    print("cmd:", cmd)
    if cmd == HELP_CMD:
        help_menu() # help() is python builtin :/
    else:
        for var, val in list(globals().items()): # iterate through global variables
            if cmd == var:
                print(val)

def fuse_thread():
    global filesystem
    filesystem = Memory((bootstrap_ip, args.port), listen_sock.getsockname())
    fuse = FUSE(filesystem, args.mount, foreground=True)

def sigint_handler(signal2,frame):
    print("SIGINT~~~~~~~~~~~~~~")
    leave()
    #LEAVE HERE???
    os.kill(os.getpid(),signal.SIGKILL)
    sys.exit(0)

if __name__ == "__main__":
    signal.signal(signal.SIGINT,sigint_handler)
    setup_args()
    print(args)
    if args.ip:
        bootstrap_ip = args.ip
    print("Starting FUSE...")
    #logging.basicConfig(level=logging.DEBUG)
    if not os.path.exists(args.mount):
        try:
            os.makedirs(args.mount)
        except:
            print("\033[91m"+"Directory " + args.mount + " is already mounted. Please unmount and try again.")
            print("To unmount, maybe try \033[1m $fusermount -uz " + args.mount)
            sys.exit(0)
    intro()
    join()
    t = threading.Thread(target=fuse_thread, name='fuse_thread')
    t.setDaemon(True)
    t.start()
    sel.register(sys.stdin, selectors.EVENT_READ, do_cmd)
    sel.register(listen_sock, selectors.EVENT_READ, accept)
    while True:
            prompt()
            events = sel.select()
            for key, mask in events: # dw about mask
                    print(key)
                    callback = key.data
                    callback(key.fileobj)