#imports
from splinter import Browser
from bs4 import BeautifulSoup as soup
import datetime as dt
from webdriver_manager.chrome import ChromeDriverManager

#scrape all function
def scrape_all():
    #set up Splinter
    executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', **executable_path, headless=False)

    #retrieve information from web page
    news_title, news_p = scrape_news(browser)

    #build a dictionary using the information from scrapes
    marsData  = {
        "newsTitle": news_title,
        "newsParagraph": news_p,
        "featuredImage": scrape_feature_img(browser),
        "facts": scrape_facts(browser),
        "hemispheres": scrape_hemispheres(browser),
        "lastUpdated": dt.datetime.now()
    }

    #stop webdriver
    browser.quit()

    return marsData

#scrape mars news pages
def scrape_news(browser):
    #Visit URL
    url = 'https://redplanetscience.com/'
    browser.visit(url)

    # Optional delay for loading the page
    browser.is_element_present_by_css('div.list_text', wait_time=1)

     #Convert the browser html to a soup object
    html = browser.html
    news_soup = soup(html, 'html.parser')

    slide_elem = news_soup.select_one('div.list_text')

    #retrieves title
    news_title = slide_elem.find("div", class_="content_title").text.strip()
    #retrieves paragraph
    news_p = slide_elem.find("div", class_="article_teaser_body").text.strip()

    #return title and paragraph
    return news_title, news_p

#scrape featured image page
def scrape_feature_img(browser):
    # Visit URL
    url = 'https://spaceimages-mars.com'
    browser.visit(url)

    # Find and click the full image button
    browser.links.find_by_partial_text("FULL IMAGE").click()

    # Parse the resulting html with soup
    space_html = browser.html
    img_soup = soup(space_html, 'html.parser')
    img_elem = img_soup.find("img", class_="headerimage")

    # find the relative image url
    img_url_rel = img_elem["src"]

    # Use the base url to create an absolute url
    img_url = url + "/" + img_url_rel

    #return image url
    return img_url

#scrape facts page
def scrape_facts(browser):
    #Visit URL
    url = 'https://galaxyfacts-mars.com/'
    browser.visit(url)

    # Parse the resulting html with soup
    facts_html = browser.html
    facts_soup = soup(facts_html, 'html.parser')

    #find facts location
    facts_location = facts_soup.find('div', class_ = "diagram mt-4")
    #retrieve the html code
    facts_table = facts_location.find("table")

    #create an empty string
    facts = ""

    #add text to empty string and return
    facts += str(facts_table)
    return facts

#scrape hemispheres pages
def scrape_hemispheres(browser):
    #Visit URL
    url = 'https://marshemispheres.com/'
    browser.visit(url)

    # Create a list to hold the images and titles.
    hemisphere_image_urls = []

    #set up for loop
    for i in range(4):
        # We have to find the elements on each loop to avoid a stale element exception
        browser.find_by_css("a.product-item img")[i].click()

        # Next, we find the Sample image anchor tag and extract the href
        sample_elem = browser.find_link_by_text("Sample").first
        img_url = sample_elem["href"]

        # Get Hemisphere title
        title = browser.find_by_css("h2.title").text

        #create dictionary to stroy data for url and title
        img_url_dict = {
            "title": title,
            "img_url": img_url
        }

        # Append hemisphere object to list
        hemisphere_image_urls.append(img_url_dict)

        # Finally, we navigate backwards
        browser.back()

    #return hemisphere URLs with titles
    return hemisphere_image_urls

#set up as flask app
if __name__ == "__main__":
    print(scrape_all())


from django.shortcuts import render
from django.views.generic import ListView, DetailView
from .models import *
from .forms import TrialFiltersForm

# Create your views here.
def homepage(request):
    clinical_trials = ContentTypeClinicalTrials.objects.filter(field_id_number_value__icontains="RPCEC").order_by(
        '-vid')
    rpcec = DatosInternosRpcec.objects.all()[0].cod_rpcec

    i = 1
    trial_list = []
    while i <= rpcec:
        j = i
        aux = 'RPCEC' + str(j).zfill(8)
        clinical_trial = clinical_trials.filter(field_id_number_value=aux)[0]
        trial_list.append(clinical_trial)
        i += 1

    context = {'trial_registrered': len(trial_list)}
    trial_anticipated = 0
    trial_actual = 0

    trial_year = {'trial_january': 0, 'trial_february': 0, 'trial_march': 0, 'trial_april': 0, 'trial_may': 0,
                  'trial_june': 0, 'trial_july': 0, 'trial_august': 0, 'trial_september': 0, 'trial_october': 0,
                  'trial_november': 0, 'trial_december': 0}

    for trial in trial_list:
        # Formatting date of registration
        if int(trial.field_id_number_value[5:]) <= 163:
            date_registration = str(trial.field_date_register_value[-2:]) + '/' + str(
                trial.field_date_register_value[5:7]) + '/' + str(trial.field_date_register_value[0:4])

        else:
            date_registration = str(trial.field_date_register_value)

        if date_registration[-4:] == '2021':
            if date_registration[-7:-5] == '01':
                trial_year['trial_january'] = trial_year['trial_january'] + 1

            if date_registration[-7:-5] == '02':
                trial_year['trial_february'] = trial_year['trial_february'] + 1

            if date_registration[-7:-5] == '03':
                trial_year['trial_march'] = trial_year['trial_march'] + 1

            if date_registration[-7:-5] == '04':
                trial_year['trial_april'] = trial_year['trial_april'] + 1

            if date_registration[-7:-5] == '05':
                trial_year['trial_may'] = trial_year['trial_may'] + 1

            if date_registration[-7:-5] == '06':
                trial_year['trial_june'] = trial_year['trial_june'] + 1

            if date_registration[-7:-5] == '07':
                trial_year['trial_july'] = trial_year['trial_july'] + 1

            if date_registration[-7:-5] == '08':
                trial_year['trial_august'] = trial_year['trial_august'] + 1

            if date_registration[-7:-5] == '09':
                trial_year['trial_september'] = trial_year['trial_september'] + 1

            if date_registration[-7:-5] == '10':
                trial_year['trial_october'] = trial_year['trial_october'] + 1

            if date_registration[-7:-5] == '11':
                trial_year['trial_november'] = trial_year['trial_november'] + 1

            if date_registration[-7:-5] == '12':
                trial_year['trial_december'] = trial_year['trial_december'] + 1

        # Formatting date enrolment
        date_enrolment = str(trial.field_date_first_enrollment_value)[8:10] + '/' + str(
            trial.field_date_first_enrollment_value)[5:7] + '/' + str(trial.field_date_first_enrollment_value)[
                                                                  0:4]
        # Setting type of enrolment
        if date_enrolment[-4:] < date_registration[-4:]:
            trial_actual = trial_actual + 1
        else:
            if date_enrolment[-4:] > date_registration[-4:]:
                trial_anticipated = trial_anticipated + 1
            else:
                if date_enrolment[-7:-5] < date_registration[-7:-5]:
                    trial_actual = trial_actual + 1
                else:
                    if date_enrolment[-7:-5] > date_registration[-7:-5]:
                        trial_anticipated = trial_anticipated + 1
                    else:
                        if date_enrolment[:2] >= date_registration[:2]:
                            trial_anticipated = trial_anticipated + 1
                        else:
                            trial_actual = trial_actual + 1

    context['trial_anticipated'] = trial_anticipated
    context['trial_actual'] = trial_actual
    context['trial_year'] = list(trial_year.values())

    return render(request, 'index.html', context)

class ClinicalTrialListView(ListView):
    model = ContentTypeClinicalTrials
    template_name = 'clinical_trial/clinical_trial_list.html'
    context_object_name = 'ClinicalTrials'

    def get_queryset(self):
        clinical_trials = ContentTypeClinicalTrials.objects.filter(field_id_number_value__icontains="RPCEC").order_by(
            '-vid')
        rpcec = DatosInternosRpcec.objects.all()[0].cod_rpcec

        i = 1
        trial_list = []

        recruitment_status = self.request.GET.get('recruitment_status_field')
        if recruitment_status:
            clinical_trials = clinical_trials.filter(field_recruitment_status_value=recruitment_status)

        phase = self.request.GET.get('phase_field')
        if phase:
            clinical_trials = clinical_trials.filter(field_phase_value=phase)

        covid_trial = self.request.GET.get('covid_trial_field')
        if covid_trial == "1":
            clinical_trials = clinical_trials.filter(field_scientific_title_value__icontains='COVID-19')

        while i <= rpcec:
            j = i
            aux = 'RPCEC' + str(j).zfill(8)
            try:
                clinical_trial = clinical_trials.filter(field_id_number_value=aux)[0]
                trial_list.append(clinical_trial)
            except:
                pass
            i += 1

        return trial_list

    def get_context_data(self, **kwargs):
        context = super(ClinicalTrialListView, self).get_context_data(**kwargs)
        context['form'] = TrialFiltersForm(initial={
            'recruitment_status_field': self.request.GET.get('recruitment_status_field'),
            'phase_field': self.request.GET.get('phase_field'),
            'covid_trial_field': self.request.GET.get('covid_trial_field'),
        })
        return context

class ClinicalTrialDetailView(DetailView):
    model = ContentTypeClinicalTrials
    template_name = 'clinical_trial/clinical_trial_detail.html'
    context_object_name = 'trial'

    def get_context_data(self, **kwargs):
        context = super(ClinicalTrialDetailView, self).get_context_data(**kwargs)

        context['object'].secondary_sponsors = ContentFieldSecSponsor.objects.filter(nid=context['object'].nid,
                                                                                vid=context['object'].vid)

        sec_ids = ContentFieldIdentifying.objects.filter(nid=context['object'].nid, vid=context['object'].vid)

        issuing_authorities = ContentFieldAuthorityIdentifying.objects.filter(nid=context['object'].nid,
                                                                              vid=context['object'].vid)
        secondary_ids = []
        for i in range(0, len(sec_ids)):
            secondary_id = {'sec_id': sec_ids[i].field_identifying_value,
                            'issuing_authorities': issuing_authorities[i].field_authority_identifying_value}
            secondary_ids.append(secondary_id)

        context['object'].secondary_ids = secondary_ids

        context['object'].principal_investigator_phone = ContentFieldInvestigatorPhone.objects.filter(
            nid=context['object'].nid,
            vid=context['object'].vid)

        context['object'].principal_investigator_email = ContentFieldEmailInvestigator.objects.filter(
            nid=context['object'].nid,
            vid=context['object'].vid)

        context['object'].countries = ContentFieldCountries.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].clinical_sites = ContentFieldParticipatingSc.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].ethics_committes = ContentFieldEthicsCommittes.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].hc_codes = ContentFieldHcCode.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].hc_keywords = ContentFieldHcKeyword.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].i_codes = ContentFieldInterCode.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].field_phone_public_value = ContentFieldPublicPhone.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].field_email_public_value = ContentFieldEmailPublic.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].field_phone_scientific_value = ContentFieldScientificPhone.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        context['object'].field_phone_scientific_value = ContentFieldEmailScientific.objects.filter(
            nid=context['object'].nid, vid=context['object'].vid)

        if context['object'].field_study_date_value:
            context['object'].field_study_date_value = str(context['object'].field_study_date_value[8:10]) + '/' + str(
                context['object'].field_study_date_value[5:7]) + '/' + str(
                context['object'].field_study_date_value[0:4])

        if context['object'].field_results_date_value:
            context['object'].field_results_date_value = str(
                context['object'].field_results_date_value[8:10]) + '/' + str(
                context['object'].field_results_date_value[5:7]) + '/' + str(
                context['object'].field_results_date_value[0:4])

        if context['object'].field_first_publication_value:
            context['object'].field_first_publication_value = str(
                context['object'].field_first_publication_value[8:10]) + '/' + str(
                context['object'].field_first_publication_value[5:7]) + '/' + str(
                context['object'].field_first_publication_value[0:4])

        ethics_reviews = ContentFieldResearchCommittee.objects.filter(nid=context['object'].nid,
                                                                      vid=context['object'].vid)

        ethics_committee = []
        for i in range(0, len(ethics_reviews)):
            ethic_committee = {}

            ethic_committee['name'] = ethics_reviews[i].field_research_committee_value
            ethics_status = ContentFieldEvaluationStatus.objects.get(nid=context['object'].nid,
                                                                     vid=context['object'].vid,
                                                                     delta=ethics_reviews[i].delta)
            ethic_committee['status'] = ethics_status.field_evaluation_status_value

            approval_date = ContentFieldDateCommittee.objects.get(nid=context['object'].nid, vid=context['object'].vid,
                                                                  delta=ethics_reviews[i].delta)
            if approval_date.field_date_committee_value:
                ethic_committee['approval_date'] = approval_date.field_date_committee_value[
                                                   8:10] + '/' + approval_date.field_date_committee_value[
                                                                 5:7] + '/' + approval_date.field_date_committee_value[
                                                                              0:4]
            else:
                ethic_committee['approval_date'] = None

            contact_address = ContentFieldZipCommittee.objects.get(nid=context['object'].nid, vid=context['object'].vid,
                                                                   delta=ethics_reviews[i].delta)
            ethic_committee['contact_address'] = contact_address.field_zip_committee_value

            try:
                contact_phone = ContentFieldCommmitteFhone.objects.get(nid=context['object'].nid,
                                                                       vid=context['object'].vid,
                                                                       delta=ethics_reviews[i].delta)
            except ContentFieldCommmitteFhone.DoesNotExist:
                contact_phone.field_commmitte_fhone_value = '-'

            ethic_committee['contact_phone'] = contact_phone.field_commmitte_fhone_value

            try:
                contact_email = ContentFieldCommmitteMail.objects.get(nid=context['object'].nid,
                                                                      vid=context['object'].vid,
                                                                      delta=ethics_reviews[i].delta)
            except ContentFieldCommmitteMail.DoesNotExist:
                contact_email.field_commmitte_mail_value = '-'

            ethic_committee['contact_email'] = contact_email.field_commmitte_mail_value

            ethics_committee.append(ethic_committee)

        context['object'].ethics_committee = ethics_committee
        return context


import numpy as np
from PIL import Image, ImageDraw
import os
from detr import detr

if __name__ == '__main__':

    data_dir = "../coco/val/"

    # # padded data
    # max_h, max_w = 0, 0
    # img_lst = []
    # for file in os.listdir(data_dir):
    #     img = Image.open(os.path.join(data_dir, file))
    #     w, h = img.size
    #     ratio = min(1333/max(w,h), 800/min(w,h))
    #     new_w, new_h = int(w*ratio), int(h*ratio)
    #     img = np.array(img.resize((new_w,new_h))) / 255.
    #     img_lst.append(img)
    #     max_h = max(max_h, new_h)
    #     max_w = max(max_w, new_w)
    # print('max_h', max_h, 'max_w', max_w)

    # batch_lst = []
    # for img in img_lst:
    #     h, w, c = img.shape
    #     pad_h, pad_w = max_h-h, max_w-w
    #     img = np.pad(img, [[0,pad_h],[0,pad_w],[0,0]], mode='constant')
    #     batch_lst.append(img)
    # batch_inpt = np.stack(batch_lst, axis=0)   # [b,h,w,c]

    # model
    model = detr(input_shape=(800,1200,3), pe='sine', n_classes=92, depth=50, dilation=False,
                 emb_dim=256, enc_layers=6, dec_layers=6, max_boxes=100, mode='test')
    model.load_weights("weights/detr-r50.h5")

    # run prediction
    for file in os.listdir(data_dir):
        img = Image.open(os.path.join(data_dir,file))

        inpt = np.array(img.resize((1200,800)))
        inpt = np.expand_dims(inpt/255., axis=0)   # [1,h,w,3]
        mask = np.zeros((1,800,1200,1))    # img_mask, mask=1 represents useless edges

        preds = model.predict([inpt,mask])
        cls_preds = preds[0][0,:,:-1]    # fg scores, [N,cls]
        box_preds = preds[1][0]   # [N,4]

        scores = np.max(cls_preds, axis=-1)    # [N,]
        labels = np.argmax(cls_preds, axis=-1)  # [N,]

        boxes_xcycwh = box_preds  # [N,4], 0-1 normed, xcycwh wrt. origin wh
        boxes_x1y1x2y2 = np.stack([boxes_xcycwh[:,0]-boxes_xcycwh[:,2]/2,   # [N,4], normed x1y1x2y2
                                   boxes_xcycwh[:,1]-boxes_xcycwh[:,3]/2,
                                   boxes_xcycwh[:,0]+boxes_xcycwh[:,2]/2,
                                   boxes_xcycwh[:,1]+boxes_xcycwh[:,3]/2], axis=1)

        # draw fg boxes
        N = scores.shape[0]
        draw = ImageDraw.Draw(img)
        w, h = img.size
        for i in range(N):
            if scores[i]>0.5:
                x1,y1,x2,y2 = boxes_x1y1x2y2[i]*np.array([w,h,w,h]).tolist()
                draw.rectangle((x1,y1,x2,y2), fill=None, outline=(0,0,0), width=5)
        img.save('preds/'+file)


import sys

from pig import views as pig_views

if sys.version_info[0] > 2:
  from django.urls import re_path
else:
  from django.conf.urls import url as re_path

urlpatterns = [
  re_path(r'^$', pig_views.app, name='index'),

  re_path(r'^app/?$', pig_views.app, name='app'),

  # Ajax
  re_path(r'^scripts/?$', pig_views.scripts, name='scripts'),
  re_path(r'^dashboard/?$', pig_views.dashboard, name='dashboard'),
  re_path(r'^save/?$', pig_views.save, name='save'),
  re_path(r'^run/?$', pig_views.run, name='run'),
  re_path(r'^copy/?$', pig_views.copy, name='copy'),
  re_path(r'^delete/?$', pig_views.delete, name='delete'),
  re_path(r'^watch/(?P<job_id>[-\w]+)$', pig_views.watch, name='watch'),
  re_path(r'^stop/?$', pig_views.stop, name='stop'),
  re_path(r'^install_examples$', pig_views.install_examples, name='install_examples'),
]


import os
from subprocess import call
import subprocess
import shlex
import string
import distutils.core
import shutil

def uploadToDropBox(currDir):
    for root,dirs,fileNames in os.walk(currDir):
        for f in fileNames:
            currFilePath = os.path.join(root,f)
            if (('tex' in currFilePath or 'pdf' in currFilePath or 'result' in currFilePath) and 'ACA' not in currFilePath and './' not in currFilePath and 'log' not in currFilePath):
                src = currFilePath;
                if 'boundaryLR' in currFilePath:
                    dst = '/Users/Amir/Dropbox/Amir_Siva/'+currFilePath[string.find(currFilePath,'benchmarks/')+11:]
                else:
                    dst = '/Users/Amir/Dropbox/Amir_Siva/boundaryLR/'+currFilePath[string.find(currFilePath,'benchmarks/')+11:]
                shutil.copyfile(src,dst)

def getTime(stream,token,startIdx):
    timeStartIdx = string.find(stream,token,startIdx);
    timeEndIdx   = string.find(stream,'seconds',timeStartIdx);
    totalTime = string.split(stream[timeStartIdx:timeEndIdx - 1])
    totalTime = totalTime[len(totalTime) - 1];
    return float(totalTime);

def getInfo(stream,token,startIdx):
    infoStartIdx = string.find(stream,token,startIdx);
    infoEndIdx   = string.find(stream,'\n',infoStartIdx)
    info         = stream[infoStartIdx:infoEndIdx]
    info         = string.split(info)
    info         = info[len(info) - 1]
    return info

def insertNumPoints(filePath,numPointsVec):
    f    = open(filePath)
    data = f.readlines()
    f.close();
    f = open(filePath,'w')
    for i in range(len(numPointsVec)):
        f.write(string.split(numPointsVec[i])[1] +'\t'+ string.split(data[i])[1]+'\n')
    f.close();

def parseResultFile(path,nameList):
    f = open(path)
    data = f.read()
    f.close();

    size = {}
    directAccuracy = {}
    directTime     = {}
    finalAccuracy  = {}
    totalTime  = {}
    LUTime     = {}
    numIter    = {}
    for matrixPath in nameList:
        matrix = matrixPath[matrixPath.find('input/') + 6:]
        startIdx = string.find(data,matrix)

        size[matrixPath]          = int(getInfo(data,'Matrix Size',startIdx))
        numIter[matrixPath]       = int(getInfo(data,'Number of Iterations',startIdx))
        finalAccuracy[matrixPath] = float(getInfo(data,'Residual l2 Relative Error',startIdx))

        directAccuracyStartIdx = startIdx + len(matrix) + 3
        directAccuracyEndIdx   = string.find(data,'\n',directAccuracyStartIdx)
        directAccuracy[matrixPath] = float(data[directAccuracyStartIdx:directAccuracyEndIdx])

        totalTime[matrixPath] = getTime(data,'Total Solve Time',startIdx)
        directTime[matrixPath] = getTime(data,'HODLR Direct Solver Total Time',startIdx)
        LUTime[matrixPath] = getTime(data,'LU Solve Time',startIdx)

    return size,directAccuracy,directTime,finalAccuracy,totalTime,LUTime,numIter

def correctTimingCaption(inputFilePath,size):
    matrixName       = inputFilePath[inputFilePath.find('input/') + 6:]
    timingFileName   = matrixName + '_Timing_Plot.tex';
    root = inputFilePath[0:inputFilePath.find('input/')]
    timingFolderPath = root + 'timing_Plots/'
    timingFilePath   = timingFolderPath + timingFileName;
    f = open(timingFilePath)
    data = f.read()
    f.close()
    startIdx = string.find(data,'size of');
    endIdx   = string.find(data,'.',startIdx);
    correctSize = size[inputFilePath];
    correctData = data[:startIdx+8]+str(correctSize)+data[endIdx:]
    f = open(timingFilePath,'w')
    f.write(correctData)
    f.close()
    os.chdir(timingFolderPath)
    proc = subprocess.Popen(shlex.split('pdflatex ' + timingFileName))
    proc.communicate()
    os.unlink(timingFileName[:-4] + '.aux')
    os.unlink(timingFileName[:-4] + '.log')

def getMatrixInfo(inputFilePath):
    matrixName = inputFilePath[inputFilePath.find('input/') + 6:]
    nameSplit  = string.split(matrixName,'_');
    if 'blade' in matrixName:
        matrixLevel = 'top'
    else:
        level = int(nameSplit[5]);
        if level == 0:
            matrixLevel = 'top'
        elif level == 1:
            matrixLevel = 'first'
        elif level == 2:
            matrixLevel = 'second'
        else:
            matrixLevel = str(level)+'th'

    if 'blade' in matrixName:
        if '9' in matrixName:
            sparseSize = '642K'
        if '12' in matrixName:
            sparseSize = '1.26M'
        if '16' in matrixName:
            sparseSize = '2.14M'
    else:
        sparseSize = str(int(nameSplit[0]))+'K'

    if 'FETI' in inputFilePath:
        matrixType = 'FETI local sparse'
    elif 'stiffness' in inputFilePath:
        matrixType = 'stiffness';

    if '/structured/' in inputFilePath:
        meshType = 'a structured'
    elif '/unstructured' in inputFilePath:
        meshType = 'an unstructured'

    if '2D' in inputFilePath:
        meshDim = '2D'
    else:
        meshDim = '3D'

    return matrixLevel,sparseSize,matrixType,meshType,meshDim

def generateLRAnalysisFigure(inputFilePath):
    root = inputFilePath[0:inputFilePath.find('input/')]
    LRFolderPath = root + 'LR_Analysis_Plots/'
    matrixName   = inputFilePath[inputFilePath.find('input/') + 6:]
    LRFileName   = matrixName + '_LR_Plot.tex';
    SVDFileName  = matrixName + '_SingularValueDecay';
    distanceFileName_Col       = matrixName + '_distanceFromBoundaryVsPivotSize_Col'
    distanceFileName_Row       = matrixName + '_distanceFromBoundaryVsPivotSize_Row'
    SVDErrorFileName           = matrixName + '_SVDErrorVsBoundaryDistance'
    boundaryErrorFileName      = matrixName + '_boundaryErrorVsBoundaryDistance'
    boundaryErrorCompFileName1 = matrixName + '_boundaryErrorCompVsBoundaryDistance1'
    boundaryErrorCompFileName3 = matrixName + '_boundaryErrorCompVsBoundaryDistance3'
    boundaryErrorCompFileName5 = matrixName + '_boundaryErrorCompVsBoundaryDistance5'
    numPointsFileName          = matrixName + '_numPointsVsBoundaryDistance'
    SVDFilePath                = root + 'results_LR/' + SVDFileName;
    distanceFilePath_Col       = root + 'results_LR/' + distanceFileName_Col;
    distanceFilePath_Row       = root + 'results_LR/' + distanceFileName_Row;
    SVDErrorFilePath           = root + 'results_LR/' + SVDErrorFileName;
    boundaryErrorFilePath      = root + 'results_LR/' + boundaryErrorFileName;
    boundaryErrorCompFilePath1 = root + 'results_LR/' + boundaryErrorCompFileName1;
    boundaryErrorCompFilePath3 = root + 'results_LR/' + boundaryErrorCompFileName3;
    boundaryErrorCompFilePath5 = root + 'results_LR/' + boundaryErrorCompFileName5;
    LRFilePath                 = LRFolderPath + LRFileName;
    numPointsFilePath          = root + 'results_LR/' + numPointsFileName;
    width = str(8)

    numPoints_f = open(numPointsFilePath)
    numPoints   = numPoints_f.readlines();
    numPoints_f.close();

    insertNumPoints(SVDErrorFilePath,numPoints)
    insertNumPoints(boundaryErrorFilePath,numPoints)
    insertNumPoints(boundaryErrorCompFilePath1,numPoints)
    insertNumPoints(boundaryErrorCompFilePath3,numPoints)
    insertNumPoints(boundaryErrorCompFilePath5,numPoints)
    #SVDError_f = open(SVDErrorFilePath)
    #SVDError   = SVDError_f.readlines()
    #SVDError_f.close();

    #SVDError_f = open(SVDErrorFilePath,'w')
    #for i in range(len(numPoints)):
    #    SVDError_f.write(string.split(numPoints[i])[1] +'\t'+ string.split(SVDError[i])[1]+'\n')
    #SVDError_f.close();

    #boundaryError_f = open(boundaryErrorFilePath)
    #boundaryError   = boundaryError_f.readlines()
    #boundaryError_f.close();

    #boundaryError_f = open(boundaryErrorFilePath,'w')
    #for i in range(len(numPoints)):
    #    boundaryError_f.write(string.split(numPoints[i])[1] +'\t'+ string.split(boundaryError[i])[1]+'\n')
    #boundaryError_f.close();

    #boundaryErrorComp_f = open(boundaryErrorCompFilePath1)
    #boundaryErrorComp   = boundaryErrorComp_f.readlines()
    #boundaryErrorComp_f.close();

    #boundaryErrorComp_f = open(boundaryErrorCompFilePath1,'w')
    #for i in range(len(numPoints)):
    #    boundaryErrorComp_f.write(string.split(numPoints[i])[1] +'\t'+ string.split(boundaryErrorComp[i])[1]+'\n')
    #boundaryErrorComp_f.close();

    SVD_f = open(SVDFilePath)
    denseSize = str(len(SVD_f.readlines()));
    SVD_f.close();

    f = open(LRFilePath,'w');
    f.write('\\documentclass[12pt,letterpaper]{article}\n')
    f.write('\n')
    f.write('\\usepackage{tikz}\n')
    f.write('\\usepackage{graphicx,subfigure}\n')
    f.write('\\usepackage{pgfplots}\n')
    f.write('\\usepackage[margin=0.5in]{geometry}\n')
    f.write('\\pgfplotsset{compat=1.3}\n')
    f.write('\n')
    f.write('\\begin{document}\n')
    f.write('\\pagenumbering{gobble}\n')
    f.write('\\begin{figure}\n')
    f.write('\\centering\n')
    f.write('\\subfigure[Row distance from boundary vs pivot size]{\n');
    f.write('\t\\begin{tikzpicture}\n');
    f.write('\t\t\\pgfplotsset{every axis legend/.append style={ at={(0.5,1.03)},anchor=south}}\n')
    f.write('\t\t\\begin{axis}[width='+width+'cm,height=7cm,grid=major,xlabel=Pivot size (largest to smallest),ylabel=Row distance from boundary]\n');
    f.write('\t\t\t\\addplot file{"'+distanceFilePath_Row+'"};\n');
    f.write('\t\t\\end{axis}\n')
    f.write('\t\\end{tikzpicture}\n')
    f.write('}\n')
    f.write('\\subfigure[Col distance from boundary vs pivot size]{\n');
    f.write('\t\\begin{tikzpicture}\n');
    f.write('\t\t\\pgfplotsset{every axis legend/.append style={ at={(0.5,1.03)},anchor=south}}\n')
    f.write('\t\t\\begin{axis}[width='+width+'cm,height=7cm,grid=major,xlabel=Pivot size (largest to smallest),ylabel=Col distance from boundary]\n');
    f.write('\t\t\t\\addplot file{"'+distanceFilePath_Col+'"};\n');
    f.write('\t\t\\end{axis}\n')
    f.write('\t\\end{tikzpicture}\n')
    f.write('}\n')
    f.write('\\subfigure[Singular value decay]{\n');
    f.write('\t\\begin{tikzpicture}\n');
    f.write('\t\t\\pgfplotsset{every axis legend/.append style={ at={(0.5,1.03)},anchor=south}}\n')
    f.write('\t\t\\begin{semilogyaxis}[width='+width+'cm,height=7cm,grid=major,xlabel=Singular value index, ylabel=Singular values]\n');
    f.write('\t\t\t\\addplot file{"'+SVDFilePath+'"};\n')
    f.write('\t\t\\end{semilogyaxis}\n');
    f.write('\t\\end{tikzpicture}\n');
    f.write('}\n')
    f.write('\\subfigure[Error vs NumPoints]{\n');
    f.write('\t\\begin{tikzpicture}\n');
    f.write('\t\t\\begin{semilogyaxis}[width='+width+'cm,height=7cm,grid=major,xlabel=Number of Added Points, ylabel=Relative Error,legend style={font=\\tiny}]\n');
    f.write('\t\t\t\\addplot file{"'+SVDErrorFilePath+'"};\n')
    f.write('\t\t\t\\addplot file{"'+boundaryErrorFilePath+'"};\n')
    f.write('\t\t\t\\addplot file{"'+boundaryErrorCompFilePath1+'"};\n')
    f.write('\t\t\t\\addplot file{"'+boundaryErrorCompFilePath3+'"};\n')
    f.write('\t\t\t\\addplot file{"'+boundaryErrorCompFilePath5+'"};\n')
    f.write('\t\t\t\\legend{SVD,RRLU\\_Boundary\\_1e-15,RRLU\\_Boundary\\_1e-1,RRLU\\_Boundary\\_1e-3,RRLU\\_Boundary\\_1e-5}\n')
    f.write('\t\t\\end{semilogyaxis}\n');
    f.write('\t\\end{tikzpicture}\n');
    f.write('}\n')
    matrixLevel,sparseSize,matrixType,meshType,meshDim = getMatrixInfo(inputFilePath)
    f.write('\\caption{Figures are for the top off-diagonal block of the '+matrixLevel+' level frontal matrix of a '+sparseSize+' '+matrixType+' matrix corresponding to '+meshType+' '+meshDim+' mesh. The off diagonal matrix has a size of '+denseSize+'.}\n')
    f.write('\\end{figure}\n')
    f.write('\\end{document}\n')
    f.close();
    os.chdir(LRFolderPath)
    proc = subprocess.Popen(shlex.split('pdflatex ' + LRFileName))
    proc.communicate()
    os.unlink(LRFileName[:-4] + '.aux')
    os.unlink(LRFileName[:-4] + '.log')

def generateTimeAnalysisFigure(inputFilePath):
    root = inputFilePath[0:inputFilePath.find('input/')]
    timingFolderPath = root + 'timing_Plots/'
    matrixName       = inputFilePath[inputFilePath.find('input/') + 6:]
    timingFileName   = matrixName + '_Timing_Plot.tex';
    timingFilePath   = timingFolderPath + timingFileName;
    recLUFactorFileName  = matrixName + '_recLU_FactorTiming'
    recLUSolveFileName   = matrixName + '_recLU_SolveTiming'
    LRTimingFileName     = matrixName + '_LR_Timing'
    levelRankFileName    = matrixName + '_levelRankAverage'
    iterTimingFileName   = matrixName + '_iter_IterTiming'
    iterAccuracyFileName = matrixName + '_iter_Accuracy'
    recLUFactorFilePath  = root + 'results_timing/' + recLUFactorFileName;
    recLUSolveFilePath   = root + 'results_timing/' + recLUSolveFileName;
    LRTimingFilePath     = root + 'results_timing/' + LRTimingFileName;
    levelRankFilePath    = root + 'results_timing/' + levelRankFileName;
    iterTimingFilePath   = root + 'results_timing/' + iterTimingFileName;
    iterAccuracyFilePath = root + 'results_timing/' + iterAccuracyFileName;
    width     = str(16)
    iterWidth = str(16)

    recLUFactor_f = open(recLUFactorFilePath)
    lines         = recLUFactor_f.readlines();
    numLevels     = int(string.split(lines[len(lines) - 1])[0])
    val = float(string.split(lines[len(lines) - 2])[1])
    recLUFactor_f.close();

    if val > 0:
        recLUFactor_f = open(recLUFactorFilePath,'a')
        recLUFactor_f.write(str(numLevels + 1) +' ' +'0\n')
        recLUFactor_f.close();

    recLUSolve_f = open(recLUSolveFilePath)
    lines        = recLUSolve_f.readlines();
    numLevels    = int(string.split(lines[len(lines) - 1])[0])
    val = float(string.split(lines[len(lines) - 2])[1])
    recLUSolve_f.close();

    if val > 0:
        recLUSolve_f = open(recLUSolveFilePath,'a')
        recLUSolve_f.write(str(numLevels + 1) +' ' +'0\n')
        recLUSolve_f.close();

    LRTiming_f = open(LRTimingFilePath)
    lines      = LRTiming_f.readlines();
    numLevels  = int(string.split(lines[len(lines) - 1])[0])
    val = float(string.split(lines[len(lines) - 2])[1])
    LRTiming_f.close();

    if val > 0:
        LRTiming_f = open(LRTimingFilePath,'a')
        LRTiming_f.write(str(numLevels + 1) +' ' +'0\n')
        LRTiming_f.close();

    iter_f  = open(iterAccuracyFilePath)
    lines   = iter_f.readlines();
    numIter = str(len(lines) - 1)
    iter_f.close();

    f = open(timingFilePath,'w');
    f.write('\\documentclass[12pt,letterpaper]{article}\n')
    f.write('\n')
    f.write('\\usepackage{tikz}\n')
    f.write('\\usepackage{graphicx,subfigure}\n')
    f.write('\\usepackage{pgfplots}\n')
    f.write('\\usepackage[margin=0.5in]{geometry}\n')
    f.write('\\pgfplotsset{compat=1.3}\n')
    f.write('\n')
    f.write('\\begin{document}\n')
    f.write('\\pagenumbering{gobble}\n')
    f.write('\\begin{figure}\n')
    f.write('\\subfigure[Detail timings and rank information for direct HODLR solver]{\n')
    f.write('\t\\begin{tikzpicture}\n')
    f.write('\t\\pgfplotsset{every axis legend/.append style={at={(0.5,1.03)},anchor=south},y axis style/.style={yticklabel style=#1,ylabel style=#1,y axis line style=#1,ytick style=#1}}\n')
    f.write('\t\t\\begin{axis}[scale only axis, axis y line*=left,ybar,ybar interval = 0.7,enlargelimits=0.05,width = '+width+'cm, height =7cm,legend style={at={(0.5,1.03)},anchor=south,legend columns=3},ylabel={Time (s)},xlabel=HODLR Level]\n')
    f.write('\t\t\t\\addplot table [x index=0,y index=1,header=false] {' +recLUFactorFilePath + '};\n')
    f.write('\t\t\t\\addplot table [x index=0,y index=1,header=false] {' +recLUSolveFilePath + '};\n')
    f.write('\t\t\t\\addplot table [x index=0,y index=1,header=false] {' +LRTimingFilePath + '};\n')
    f.write('\t\t\t\\legend{Factorization, Solve, Low-Rank Approximation}\n')
    f.write('\t\t\\end{axis}\n')
    f.write('\t\t\\begin{axis}[scale only axis, axis y line*=right,y axis style=blue!75!black,width='+width+'cm, height =7cm,ylabel=Average Rank,xtick=\empty]\n')
    f.write('\t\t\t\\addplot [line legend, thick, sharp plot, stack plots=false,color=blue,mark=*] table[y index =1]{'+levelRankFilePath+'};\n');
    f.write('\t\t\\end{axis}\n')
    f.write('\t\\end{tikzpicture}\n')
    f.write('}\n')
    f.write('\\subfigure[Iteration timing and accuracy ]{\n')
    f.write('\t\\begin{tikzpicture}\n')
    f.write('\t\\pgfplotsset{y axis style/.style={yticklabel style=#1,ylabel style=#1,y axis line style=#1,ytick style=#1}}\n')
    f.write('\t\t\\begin{axis}[scale only axis, axis y line*=left,y axis style=blue!75!black,width='+iterWidth+'cm,height=7cm,grid=major,xlabel=Iteration,ylabel=Time(s),xtick=data,legend columns=4,xmin=0,xmax='+numIter+']\n')
    f.write('\t\t\t\\addplot file{"'+iterTimingFilePath +'"};\n');
    f.write('\t\t\\end{axis}\n')
    f.write('\t\t\\begin{semilogyaxis}[scale only axis, axis y line*=right,y axis style=red!75!black,width='+iterWidth+'cm,height=7cm,ylabel=Relative Error,xtick=data,xmin=0,xmax='+numIter+']\n')
    f.write('\t\t\t\\addplot[color = red, mark = *] file{"'+iterAccuracyFilePath +'"};\n')
    f.write('\t\t\\end{semilogyaxis}\n')
    f.write('\t\\end{tikzpicture}\n')
    f.write('}\n')
    matrixLevel,sparseSize,matrixType,meshType,meshDim = getMatrixInfo(inputFilePath)
    f.write('\t\\caption{Figures are for the '+ matrixLevel +' level frontal matrix of a '+sparseSize+' '+matrixType+' matrix corresponding to '+meshType+' '+meshDim+' mesh. The dense frontal matrix has a size of 7K. Iteration 0 on the iteration plot corresponds to the direct HODLR solver accuracy. The pivot threshold for the direct solver is 0.1. That is, in obtaining the low-rank approximations, we only keep rows and columns that correspond to pivots which have a magnitude greater than 0.1 times the magnitude of the largest pivot. In computing the low-rank approximations, we only considered the points with distance at most one from the boundary.}\n')
    f.write('\end{figure}\n')
    f.write('\end{document}\n')
    f.close()
    os.chdir(timingFolderPath)
    proc = subprocess.Popen(shlex.split('pdflatex ' + timingFileName))
    proc.communicate()
    os.unlink(timingFileName[:-4] + '.aux')
    os.unlink(timingFileName[:-4] + '.log')

currDir =  os.getcwd()
# copy results from source dir
dst = currDir+'/boundaryLR';
idx = currDir.find('benchmarks')
src = currDir[:idx]+'build/'+currDir[idx:]+'/boundaryLR';
#distutils.dir_util.copy_tree(src,dst)
nameList = []

# loop through all the files
for root,dirs,fileNames in os.walk(currDir):
    for f in fileNames:
        currFilePath = os.path.join(root,f)
        if 'input/' in currFilePath:
            currResultPath = currFilePath[0:currFilePath.find('input/')]
            timingFolderPath = currResultPath + 'timing_Plots';
            LRFolderPath     = currResultPath + 'LR_Analysis_Plots';
            if not os.path.exists(LRFolderPath):
                os.makedirs(LRFolderPath)
            if not os.path.exists(timingFolderPath):
                os.makedirs(timingFolderPath)
            if ('Graph' not in f):
                if ('level' in f):
                    nameList.append(currFilePath);
                    generateLRAnalysisFigure(currFilePath)
                    generateTimeAnalysisFigure(currFilePath);

# copy output from source dir
src = currDir[:idx]+'build/'+currDir[idx:]+'/results'
dst = currDir+'/results'
#shutil.copyfile(src,dst);
size,directAccuracy,directTime,finalAccuracy,totalTime,LUTime,numIter = parseResultFile(dst,nameList);
# Open summary table file
sumFile = open(currDir+'/summary.tex','w')
sumFile.write('\\documentclass[12pt,letterpaper]{article}\n')
sumFile.write('\n')
sumFile.write('\\usepackage{tikz}\n')
sumFile.write('\\usepackage{graphicx,subfigure}\n')
sumFile.write('\\usepackage{pgfplots}\n')
sumFile.write('\\usepackage[margin=0.5in]{geometry}\n')
sumFile.write('\\pgfplotsset{compat=1.3}\n')
sumFile.write('\\usepackage{multirow}\n')
sumFile.write('\n')
sumFile.write('\\begin{document}\n')
sumFile.write('\\begin{table}\n')
sumFile.write('\t\\scalebox{0.75}{\n')
sumFile.write('\t\\begin{tabular}{|c|c|c|c||c|c|c|c|c|c|c|c|c|}\n')
sumFile.write('\t\\hline\n')
sumFile.write('\t Matrix & Mesh & Mesh & Elimination & \\multicolumn{2}{c}{Matrix Size}\\vline & \\multicolumn{2}{c}{Direct Solver}\\vline & \\multicolumn{3}{c}{Iterative Solver} \\vline&  Partial Pivoting & Percent\\\  \\cline{5-12}\n')
sumFile.write('\t Type & Type & Dim & Tree Level & Sparse & Dense& Time & Accuracy & Time & Accuracy & num Iterations &LU Time & Speedup\\\ \\hline\n')
# Loop over all files and fill the table
for root,dirs,fileNames in os.walk(currDir):
    for f in fileNames:
        currFilePath = os.path.join(root,f)
        if 'input/' in currFilePath:
            if ('Graph' not in f):
                if ('level' in f):
                    # Correct caption
                    correctTimingCaption(currFilePath,size)

                    # Write to the table file
                    matrixLevel,sparseSize,matrixType,meshType,meshDim = getMatrixInfo(currFilePath)
                    if 'FETI' in matrixType:
                        matrixType = 'FETI local'
                    if 'an' in meshType:
                        meshType = meshType[3:]
                    else:
                        meshType = meshType[2:]

                    #speedup = 100.*(LUTime[currFilePath])/totalTime[currFilePath]
                    speedup = (LUTime[currFilePath])/totalTime[currFilePath]
                    sumFile.write('\t'+matrixType+'&'+meshType+'&'+meshDim+'&'+matrixLevel+'&'+sparseSize+'&'+str(size[currFilePath])+'&'+'%.2e' %directTime[currFilePath]+'&'+'%.2e' %directAccuracy[currFilePath]+'&'+'%.2e' %totalTime[currFilePath]+'&'+'%.2e' %finalAccuracy[currFilePath]+'&'+str(numIter[currFilePath])+'&'+'%.2e' %LUTime[currFilePath]+'&'+ '%.2f' %speedup+'\\\ \n')

sumFile.write('\t\\hline\n')
sumFile.write('\t\\end{tabular}}\n')
sumFile.write('\t\\end{table}\n')
sumFile.write('\\end{document}\n')
sumFile.close()
os.chdir(currDir)
proc = subprocess.Popen(shlex.split('pdflatex ' + 'summary.tex'))
proc.communicate()
os.unlink('summary.aux')
os.unlink('summary.log')

#uploadToDropBox(currDir)


import ftrack_api.inspection
import ftrack_api.symbol

def test_identity(user):
    '''Retrieve identity of *user*.'''
    identity = ftrack_api.inspection.identity(user)
    assert identity[0] == 'User'
    assert identity[1] == ['d07ae5d0-66e1-11e1-b5e9-f23c91df25eb']

def test_primary_key(user):
    '''Retrieve primary key of *user*.'''
    primary_key = ftrack_api.inspection.primary_key(user)
    assert primary_key == {
        'id': 'd07ae5d0-66e1-11e1-b5e9-f23c91df25eb'
    }

def test_created_entity_state(session, unique_name):
    '''Created entity has CREATED state.'''
    new_user = session.create('User', {'username': unique_name})
    assert ftrack_api.inspection.state(new_user) is ftrack_api.symbol.CREATED

    # Even after a modification the state should remain as CREATED.
    new_user['username'] = 'changed'
    assert ftrack_api.inspection.state(new_user) is ftrack_api.symbol.CREATED

def test_retrieved_entity_state(user):
    '''Retrieved entity has NOT_SET state.'''
    assert ftrack_api.inspection.state(user) is ftrack_api.symbol.NOT_SET

def test_modified_entity_state(user):
    '''Modified entity has MODIFIED state.'''
    user['username'] = 'changed'
    assert ftrack_api.inspection.state(user) is ftrack_api.symbol.MODIFIED

def test_deleted_entity_state(session, user):
    '''Deleted entity has DELETED state.'''
    session.delete(user)
    assert ftrack_api.inspection.state(user) is ftrack_api.symbol.DELETED

def test_post_commit_entity_state(session, unique_name):
    '''Entity has NOT_SET state post commit.'''
    new_user = session.create('User', {'username': unique_name})
    assert ftrack_api.inspection.state(new_user) is ftrack_api.symbol.CREATED

    session.commit()

    assert ftrack_api.inspection.state(new_user) is ftrack_api.symbol.NOT_SET

def test_states(session, unique_name, user):
    '''Determine correct states for multiple entities.'''
    # NOT_SET
    user_a = session.create('User', {'username': unique_name})
    session.commit()

    # CREATED
    user_b = session.create('User', {'username': unique_name})
    user_b['username'] = 'changed'

    # MODIFIED
    user_c = user
    user_c['username'] = 'changed'

    # DELETED
    user_d = session.create('User', {'username': unique_name})
    session.delete(user_d)

    # Assert states.
    states = ftrack_api.inspection.states([user_a, user_b, user_c, user_d])

    assert states == [
        ftrack_api.symbol.NOT_SET,
        ftrack_api.symbol.CREATED,
        ftrack_api.symbol.MODIFIED,
        ftrack_api.symbol.DELETED
    ]

def test_states_for_no_entities():
    '''Return empty list of states when no entities passed.'''
    states = ftrack_api.inspection.states([])
    assert states == []

def test_skip_operations_for_non_inspected_entities(session, unique_name):
    '''Skip operations for non inspected entities.'''
    user_a = session.create('User', {'username': unique_name + '-1'})
    user_b = session.create('User', {'username': unique_name + '-2'})

    states = ftrack_api.inspection.states([user_a])
    assert states == [ftrack_api.symbol.CREATED]


ax = plt.subplot(111)

ax.set_title(objtitle)
ax.set_xlabel("$\psi$ ($\circ$)")
ax.set_ylabel("a (au)")
ax.set_xlim(0,360)
ax.set_xticks([0,60,120,180,240,300,360])

ax.set_ylim(ymin,ymax)

alltp = pd.read_csv(filename, header=None, index_col=False,skiprows=1,
                       names=["stab", "p", "q", "id", "t", "a", "psi", "e", "inc"])

#completed at least 1000 resonant cycles in full simulations
stable_particles = alltp[alltp["stab"]==1]
#completed at least 750 resonant cycles in full simulations
quasistable_particles = alltp[alltp["stab"]==2]
#did not librate for at least 750 cycles in original targeted resonance
#could still be in higher-order resonances, but mostly not
particles = alltp[alltp["stab"]==0]

#add resonance labels to the plot for anything with stable or quasi-stable
#resonant particles
first_rows = quasistable_particles[quasistable_particles["t"]==0]
aresvals = []
for index, row in first_rows.iterrows():
    ares = np.power((row["p"]/row["q"]),(2./3.))*30.06
    if(ares < ymax and ares > ymin) and not(ares in aresvals):
        resl = str(int(row["p"])) + ":" + str(int(row["q"]))
        ax.text(365,ares,resl,fontsize='small')
        aresvals.append(ares)
first_rows = stable_particles[stable_particles["t"]==0]
for index, row in first_rows.iterrows():
    ares = np.power((row["p"]/row["q"]),(2./3.))*30.06
    if(ares < ymax and ares > ymin) and not(ares in aresvals):
        resl = str(int(row["p"])) + ":" + str(int(row["q"]))
        ax.text(365,ares,resl,fontsize='small')
        aresvals.append(ares)

#plot the particles:

ids = particles[particles["t"]==0].id
alreadyplotted = []
#point size and alpha
w = 0.2*pointsize
a = 0.6
for i in ids:
    #avoids duplicate plots since id numbers are repeated for different resonances
    if not(i in alreadyplotted):
        ax.plot(particles[particles["id"]==i].psi.to_numpy()*180./np.pi,
                particles[particles["id"]==i].a.to_numpy(),
                marker='o',linestyle='none',fillstyle='full',
                alpha=a,markersize=w,
                markeredgewidth=0,zorder=-10)
        alreadyplotted.append(i)

ids = quasistable_particles[quasistable_particles["t"]==0].id

alreadyplotted = []
#point size and alpha
w = 0.3*pointsize
a = 0.7
for i in ids:
    #avoids duplicate plots since id numbers are repeated for different resonances
    if not(i in alreadyplotted):
        ax.plot(quasistable_particles[quasistable_particles["id"]==i].psi.to_numpy()*180./np.pi,
                quasistable_particles[quasistable_particles["id"]==i].a.to_numpy(),
                marker='o',linestyle='none',fillstyle='full',
                alpha=a,markersize=w,
                markeredgewidth=0,zorder=-10)
        alreadyplotted.append(i)

ids = stable_particles[stable_particles["t"]==0].id

alreadyplotted = []
#point size and alpha
w = 0.5*pointsize
a = 0.9
for i in ids:
    #avoids duplicate plots since id numbers are repeated for different resonances
    if not(i in alreadyplotted):
        ax.plot(stable_particles[stable_particles["id"]==i].psi.to_numpy()*180./np.pi,
                stable_particles[stable_particles["id"]==i].a.to_numpy(),
                marker='o',linestyle='none',fillstyle='full',
                alpha=a,markersize=w,
                markeredgewidth=0,zorder=-10)
        alreadyplotted.append(i)

ax.set_rasterization_zorder(0)

x=np.zeros(5)
y=np.zeros(5)
x = [obspsi,obspsi,obspsi,obspsi,obspsi]

y1=abf-3.*da
y2=abf-da
y3=abf
y4=abf+da
y5=abf+3.*da

y = [y1,y2,y3,y4,y5]
ax.plot(x,y,marker='o',linestyle='-',c='k')
plt.subplots_adjust(left=0.2, bottom=0.15, right=0.8, top=None, wspace=None, hspace=None)
fig.savefig(figfile,dpi=300)


file0 = open('MANIFESTO OF THE COMMUNIST PARTY.txt')
article = file0.read()
paragraphs = article.split('\n')
words = []
for EachParagraph in paragraphs:
    words += EachParagraph.split(' ')
dict0 = {}
for word in words:
    try:
        dict0[word] += 1
    except KeyError:
        dict0[word] = 1
print(dict0)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 15 15:33:19 2020

@author: luca13-cmyk
"""

import pandas as pd
import numpy as np

base = pd.read_csv('credit_data.csv')

# Lida com valores invalidos (negativos) e coloca a media em seu lugar
base.loc[base.age < 0, 'age'] = 40.92

# Divide a base de dados em previsores e classe
previsores = base.iloc[:, 1:4].values
classe = base.iloc[:, 4].values

# Troca valores faltantes pela media
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(missing_values = np.nan, strategy = 'mean')
imputer = imputer.fit(previsores[:, 1:4])
previsores[:, 1:4] = imputer.transform(previsores[:, 1:4])

# Faz o calculo de escalonamento dos dados e aplica
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
previsores = scaler.fit_transform(previsores)

# Efetua a divisao da base de treinamento e a base de testes
from sklearn.model_selection import train_test_split
(previsores_treinamento,
 previsores_teste,
 classe_treinamento,
 classe_teste) = train_test_split(previsores,
                                 classe,
                                 test_size=0.25,
                                 random_state=0)

# Calculo KNN
from sklearn.neighbors import KNeighborsClassifier
classificador = KNeighborsClassifier(n_neighbors=5,
                                     metric='minkowski',
                                     p = 2)

# Efetuando o treinamento
classificador.fit(previsores_treinamento, classe_treinamento)
# Efetuando o teste
previsoes = classificador.predict(previsores_teste)

# Verificando a media de acertos
from sklearn.metrics import confusion_matrix, accuracy_score
precisao = accuracy_score(classe_teste, previsoes)
matriz = confusion_matrix(classe_teste, previsoes)

import collections
collections.Counter(classe_teste)


from models import Inventory
import uuid

def populate_inventory(data):
    new_inventory = Inventory(public_id=str(uuid.uuid4()),
                              store_id=data['store_id'],
                              product_id=data['product_id'],
                              price=data['price'],
                              quantity=data['quantity']
                              )
    return new_inventory

def allocate_data(inventory):
    inventory_data = {'public_id': inventory.public_id,
                      'store_id': inventory.store_id,
                      'product_id': inventory.product_id,
                      'quantity': inventory.quantity,
                      'price': inventory.price,
                      }
    return inventory_data

def combine_results(inventories):
    output = []
    for inventory in inventories:
        inventory_data = allocate_data(inventory)
        output.append(inventory_data)
    return output


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 19 16:55:15 2017

@author: Evan
"""

#===============
# Import modules
#===============

import os                           # for file/folder operations
import numpy as np
import numpy.random as rnd          # for random number generators
from psychopy import visual, event, core, gui, data

#==============================================
# Settings that we might want to tweak later on
#==============================================

n_files = 180
datapath = 'data'                   # directory to save data in
impath = 'C:\Python27\images'       #directory where images can be found
imlist1 = range(1,n_files+1,1)            # image names without the suffixes
imlist1 = map(str,imlist1)
imlist2 = range(1,n_files+1,1)
imlist2 = map(str,imlist2)
imlist = imlist1 + imlist2
asfx = 'a.tif'
asfx = np.tile(asfx,n_files)
asfx = asfx.tolist()
bsfx = 'b.tif'
bsfx = np.tile(bsfx,n_files)
bsfx = bsfx.tolist()
sfx = asfx + bsfx
#sfx = rnd.permutation(sfx)
combo = np.column_stack((imlist,sfx))
np.random.shuffle(combo)
imlist = combo[:,0]
imlist = imlist.tolist()
sfx = combo[:,1]
sfx = sfx.tolist()
scrsize = (1280,1024)                 # screen size in pixels
timelimit = 10                       # trial time limit in seconds

#========================================
# Store info about the experiment session
#========================================

# Get subject name, gender, age, handedness through a dialog box
exp_name = 'PF TP objects ratings'
exp_info = {
            'participant': '',
             'gender': ('male', 'female'),
            'age':'',
            'left-handed':False
            }
dlg = gui.DlgFromDict(dictionary=exp_info, title=exp_name)

# If 'Cancel' is pressed, quit
if dlg.OK == False:
    core.quit()

# Get date and time
exp_info['date'] = data.getDateStr()
exp_info['exp_name'] = exp_name

# Create a unique filename for the experiment data
if not os.path.isdir(datapath):
    os.makedirs(datapath)
data_fname = 'rate' + exp_info['participant'] + '_' + exp_info['date']
data_fname = os.path.join(datapath, data_fname)

#========================
# Prepare condition lists
#========================

# Check if all images exist
for im, suf in zip(imlist, sfx):
    if (not os.path.exists(os.path.join(impath, im+suf))):
        raise Exception('Image files not found in image folder: ' + str(im))

#===============================
# Creation of window and stimuli
#===============================

# Open a window
win = visual.Window(size=scrsize, color='white', units='pix', fullscr=True)
m = event.Mouse(win=win)
m.setVisible(False)

# Define trial text
start_message = visual.TextStim(win,
                                text="Use the number pad to indicate how typical the view of each object is. 1 = highly atypical 2 = somewhat atypical 3 = neutral 4 = somewhat typical 5 = highly typical. Press the spacebar to begin.",
                                color='blue', height=20)
trial_message = visual.TextStim(win,
                                text="1 = highly atypical 2 = somewhat atypical 3 = neutral 4 = somewhat typical 5 = highly typical",
                                color='blue', pos = (0,-300), height=20)
end_message = visual.TextStim(win,
                              text="All done! Please inform the experimenter.",
                              color='blue', height=20)

# Define bitmap stimulus (contents can still change)
bitmap = visual.ImageStim(win, size=None)

#==========================
# Define the trial sequence
#==========================

# Define a list of trials with their properties:
#   - Which image (without the suffix)
#   - Which orientation
stim_order = []
for trial_num, im, suf in zip(range(1,n_files*2+1), imlist, sfx):
    stim_order.append({'trial_num': trial_num, 'im': im, 'suf': suf})

trials = data.TrialHandler(stim_order, nReps=1, extraInfo=exp_info,
                           method='sequential', originPath=datapath)

#=====================
# Start the experiment
#=====================

# Initialize two clocks:
#   - for image change time
#   - for response time
change_clock = core.Clock()
rt_clock = core.Clock()

# Display trial start text
start_message.draw()
win.flip()

# Wait for a spacebar press to start the trial, or escape to quit
keys = event.waitKeys(keyList=['space', 'escape'])

# Run through the trials
for trial in trials:

    # Set the images, set the orientation
    im_fname = os.path.join(impath, trial['im'])
    bitmap.setImage(im_fname + trial['suf'])

    # Set the clocks to 0
    change_clock.reset()
    rt_clock.reset()

    # Empty the keypresses list
    # Leave an 'escape' press in for immediate exit
    keys = []

    # Start the trial
    # Stop trial if spacebar or escape has been pressed, or if 30s have passed
    while len(keys) == 0 and rt_clock.getTime() < timelimit:

       bitmap.draw()
       trial_message.draw()
       win.flip()
       rt_clock.reset()

    # Listen for a spacebar or escape press
       keys = event.getKeys(keyList=['1', '2', '3', '4', '5', 'escape'])
       if len(keys) > 0:
            break

    # Analyze the keypress
    if keys:
        if 'escape' in keys:
            # Escape press = quit the experiment
            break
        elif '1' in keys:
            resp = 1
            rt = rt_clock.getTime()
        elif '2' in keys:
            resp = 2
            rt = rt_clock.getTime()
        elif '3' in keys:
            resp = 3
            rt = rt_clock.getTime()
        elif '4' in keys:
            resp = 4
            rt = rt_clock.getTime()
        elif '5' in keys:
            resp = 5
            rt = rt_clock.getTime()

    else:
        # No press = failed change detection; maximal response time
        resp = None
        rt = timelimit

    # Add the current trial's data to the TrialHandler
    trials.addData('rt', rt)
    trials.addData('resp', resp)
    keys = []

    # Advance to the next trial after brief pause
    win.flip()
    core.wait(.5)

    if trial['trial_num'] == n_files*2:
        end_message.draw()
        win.flip()
        keys = event.waitKeys(keyList=['space', 'escape'])

#======================
# End of the experiment
#======================

# Save all data to a file
trials.saveAsWideText(data_fname + '.csv', delim=',')

# Quit the experiment
m.setVisible(True)
win.close()
core.quit()


import pandas as pd
import numpy as np

class Normalize:
    def __init__(self, dataset: pd.DataFrame, numerical_coefs=None) -> None:
        """
        Save normalization coefficients for numerical columns
        """
        if numerical_coefs:
            self.numerical_coefs = numerical_coefs
        else:
            numerical = dataset.select_dtypes(include=["float64", "int64"]).columns
            self.numerical_coefs = {
                column: (dataset[column].min(), dataset[column].max())
                for column in numerical
            }

    def normilize(self, dataset):
        """
        Normalise dataset:
            For numerical columns:
                new_x = (x - x_min) / (x_max - x_min)

            For no numerical columns:
                | Column_A |     | Column_A | Column_B | Column_C |
                |    A     |     |    1     |    0     |    0     |
                |    B     |  => |    0     |    1     |    0     |
                |    C     |     |    0     |    0     |    1     |
        """
        new_dataset = pd.get_dummies(
            dataset,
            columns=[
                column
                for column in dataset.columns
                if column not in self.numerical_coefs
            ],
        )

        for column, min_max in self.numerical_coefs.items():
            min, max = min_max
            new_dataset[column] = (dataset[column] - min) / (max - min)

        return new_dataset

class TrainingExamples:
    training_examples_percentage = 0.8

    def __init__(self, filename, y_column, delimiter=","):
        """
        Fetch test data from file, saves in format:
            nx         - count of columns in dataset
            m_training - count of training examples
            m_test     - count of training examples

                [x_1(1)  ...   x_1(j)  ...]
                [...     ...   ...     ...]
            x - [x_i(1)  ...   x_i(j)  ...]  (i: i-th training expample)
                [...     ...   ...     ...]
                [x_nx(1) ...   x_nx(j) ...]

            y - [[y(1), y(2), ..., y(i), ... y(m_training)]] (i = i-th training expample)

            x_test, y_test similar to x and y
        """
        x_dataset = pd.read_csv(filename, delimiter=delimiter)
        y_dataset = x_dataset.pop(y_column)

        x_dataset = Normalize(x_dataset).normilize(x_dataset)

        self.m_training = int(x_dataset.shape[0] * self.training_examples_percentage)
        self.m_test = x_dataset.shape[0] - self.m_training

        self.x = x_dataset.head(self.m_training).values.astype(float).T
        self.y = y_dataset.head(self.m_training).values.astype(float)
        self.y = self.y.reshape((self.y.shape[0], 1)).T

        self.x_test = x_dataset.tail(self.m_test).values.astype(float).T
        self.y_test = y_dataset.tail(self.m_test).values.astype(float)

        self.nx = self.x.shape[0]

class LinearRegressionModel:
    def __init__(self, nx, w=None, b=None) -> None:
        """
        w  - weights
        b  - bias
        nx - count of neurons
        """
        self.w = w or np.zeros((nx, 1))
        self.b = b or 0
        self.nx = nx

    def sigmoid(self, z):
        """
        sigmoid function that contains values in range [0, 1]
        """
        return 1 / (1 + np.exp(-z))

    def propagate(self, X, Y):
        """
        direct distraction of gradient descent.

        Using for finding current cost and dw, db (that are used for descent in train method)
        """
        m = X.shape[0]

        y_hat = self.sigmoid(np.dot(self.w.T, X) + self.b)

        cost = np.sum(-(Y * np.log(y_hat) + (1 - Y) * np.log(1 - y_hat))) / m

        dz = y_hat - Y
        dw = np.dot(X, dz.T) / m
        db = np.sum(dz) / m

        return dw, db, cost

    def train(self, X_train, Y_train, iterations=10000, alpha=0.001):
        """
        backward distraction of gradient descent

        Using, for descent further down on gradient descent function (for improve result)
        """
        cost = 0
        for _ in range(iterations):
            dw, db, cost = self.propagate(X_train, Y_train)

            self.w = self.w - alpha * dw
            self.b = self.b - alpha * db

        return cost

    def predict(self, X):
        return self.sigmoid(np.dot(self.w.T, X) + self.b)

# Load data and train model
training_data = TrainingExamples("dataset.csv", "diabetes")
model = LinearRegressionModel(training_data.nx)
model.train(training_data.x, training_data.y)

# Count error for test data
result = model.predict(training_data.x_test)
errors_count = 0
for i in range(training_data.m_test):
    errors_count += int(bool(training_data.y_test[i]) != (result[0][i] >= 0.5))

print(errors_count / training_data.m_test * 100, errors_count)


"""
Created by : Rafay Khan
Twitter: @RafayAK
"""

import numpy as np

"""
Contains a bunch of Cost functions.
This file implementations of :
    - Binary Cross_entropy Cost function
        * compute_binary_cost(Y, P_hat) -> "unstable"
        * compute_stable_bce_cost(Y, Z) -> "stable"
        * computer_keras_like_bce_cost(Y, P_hat, from_logits=False) -> stable
    - Mean Squared Error Cost function
"""

def compute_bce_cost(Y, P_hat):
    """
    This function computes Binary Cross-Entropy(bce) Cost and returns the Cost and its
    derivative.
    This function uses the following Binary Cross-Entropy Cost defined as:
    => (1/m) * np.sum(-Y*np.log(P_hat) - (1-Y)*np.log(1-P_hat))

    Args:
        Y: labels of data
        P_hat: Estimated output probabilities from the last layer, the output layer

    Returns:
        cost: The Binary Cross-Entropy Cost result
        dP_hat: gradient of Cost w.r.t P_hat

    """
    m = Y.shape[1]  # m -> number of examples in the batch

    cost = (1/m) * np.sum(-Y*np.log(P_hat) - (1-Y)*np.log(1-P_hat))
    cost = np.squeeze(cost)  # remove extraneous dimensions to give just a scalar (e.g. this turns [[17]] into 17)

    dP_hat = (1/m) * (-(Y/P_hat) + ((1-Y)/(1-P_hat)))

    return cost, dP_hat

def compute_stable_bce_cost(Y, Z):
    """
    This function computes the "Stable" Binary Cross-Entropy(stable_bce) Cost and returns the Cost and its
    derivative w.r.t Z_last(the last linear node) .
    The Stable Binary Cross-Entropy Cost is defined as:
    => (1/m) * np.sum(max(Z,0) - ZY + log(1+exp(-|Z|)))
    Args:
        Y: labels of data
        Z: Values from the last linear node

    Returns:
        cost: The "Stable" Binary Cross-Entropy Cost result
        dZ_last: gradient of Cost w.r.t Z_last
    """
    m = Y.shape[1]

    cost = (1/m) * np.sum(np.maximum(Z, 0) - Z*Y + np.log(1+ np.exp(- np.abs(Z))))
    dZ_last = (1/m) * ((1/(1+np.exp(- Z))) - Y)  # from Z computes the Sigmoid so P_hat - Y, where P_hat = sigma(Z)

    return cost, dZ_last

def compute_keras_like_bce_cost(Y, P_hat, from_logits=False):
    """
    This function computes the Binary Cross-Entropy(stable_bce) Cost function the way Keras
    implements it. Accepting either probabilities(P_hat) from the sigmoid neuron or values direct
    from the linear node(Z)

    Args:
        Y: labels of data
        P_hat: Probabilities from sigmoid function
        from_logits: flag to check if logits are being provided or not(Default: False)

    Returns:
        cost: The "Stable" Binary Cross-Entropy Cost result
        dZ_last: gradient of Cost w.r.t Z_last

    """
    if from_logits:
        # assume that P_hat contains logits and not probabilities
        return compute_stable_bce_cost(Y, Z=P_hat)

    else:
        # Assume P_hat contains probabilities. So make logits out of them

        # First clip probabilities to stable range
        EPSILON = 1e-07
        P_MAX = 1- EPSILON  # 0.9999999

        P_hat = np.clip(P_hat, a_min=EPSILON, a_max=P_MAX)

        # Second, Convert stable probabilities to logits(Z)
        Z = np.log(P_hat/(1-P_hat))

        # now call compute_stable_bce_cost
        return compute_stable_bce_cost(Y, Z)

def compute_mse_cost(Y, Y_hat):
    """
    This function computes Mean Squared Error(mse) Cost and returns the Cost and its derivative.
    This function uses the Squared Error Cost defined as follows:
    => (1/2m)*sum(Y - Y_hat)^.2

    Args:
        Y: labels of data
        Y_hat: Predictions(activations) from a last layer, the output layer

    Returns:
        cost: The Squared Error Cost result
        dY_hat: gradient of Cost w.r.t Y_hat

    """
    m = Y.shape[1]  # m -> number of examples in the batch

    cost = (1 / (2 * m)) * np.sum(np.square(Y - Y_hat))
    cost = np.squeeze(cost)  # remove extraneous dimensions to give just a scalar (e.g. this turns [[17]] into 17)

    dY_hat = -1 / m * (Y - Y_hat)  # derivative of the squared error cost function

    return cost, dY_hat


import math
import time
from typing import List

import numpy as np

from agent.agent import Agent, SOFTMAX
from agent.non_tabular.non_tabular_agent import NonTabularAgent
from agent.tabular.tabular_agent import TabularAgent
from method.actor_critic.actor_critic import get_d_ln_pi__d_theta
from method.reinforce.model.episode_vars import EpisodeVars

def reinforce_with_baseline(agent: Agent, lambda_: float,
                            alpha_weights: float, alpha_theta: float,
                            sigma: float, trial: int, trials_total: int,
                            episodes: int, trials_dir: str) -> list:
    # List of rewards across all episodes, for this one trial
    episode_returns = []

    theta = agent.init_weights()

    weights = agent.init_weights_actor_critic()

    exec_time = time.time()
    for episode in range(episodes):
        print('Episode {} / {} in Trial {} / {} (Time = {} s, '
              'Time steps = {}, {})'
            .format(episode + 1, episodes, trial + 1, trials_total,
                round(time.time() - exec_time, 2), agent.time_step, trials_dir))
        exec_time = time.time()

        agent.reset_for_new_episode(sigma=sigma)

        episode_vars_list: List[EpisodeVars] = []

        while not agent.has_terminated():
            state = agent.state

            action = agent.get_action(weights=theta,
                    action_selection_method=SOFTMAX)

            reward, state_next = agent.take_action(action)

            episode_vars_list.append(
                    EpisodeVars(state=state, state_next=state_next,
                            action=action, reward=reward)
            )

            # Episode end

        g_array = get_g_array(episode_vars_list=episode_vars_list,
                gamma=agent.gamma)

        delta_j_pi = np.zeros_like(theta)

        e_trace = agent.init_e_v()

        for t in range(len(episode_vars_list)):
            v_w, v_w_next, dv_dw = get__v_w__dv_dw(agent=agent, t=t,
                    episode_vars_list=episode_vars_list, weights=weights,
                    e_trace=e_trace)

            delta_j_pi += math.pow(agent.gamma, t) * \
                          (g_array[t] - v_w) * \
                          get_d_ln_pi__d_theta(agent=agent, theta=theta,
                                  state=episode_vars_list[t].state,
                                  action_index=agent.get_action_index(
                                          episode_vars_list[t].action)
                          )

            e_trace *= agent.gamma * lambda_ + dv_dw

            delta = episode_vars_list[t].reward + \
                    agent.gamma * v_w_next - v_w

            weights += alpha_weights * delta * e_trace

            # Time steps end

        theta += alpha_theta * delta_j_pi

        episode_returns.append(agent.returns)

        # Episode end

    return episode_returns

def get__v_w__dv_dw(agent: Agent, t: int, episode_vars_list: List[EpisodeVars],
                    weights: np.ndarray, e_trace: np.ndarray) -> tuple:
    dv_dw = np.zeros_like(e_trace)

    if isinstance(agent, TabularAgent):
        state_index = agent.get_state_index(episode_vars_list[t].state)
        state_next_index = agent.get_state_index(episode_vars_list[t].state_next)

        dv_dw[state_index] = 1

        return weights[state_index], weights[state_next_index], dv_dw

    elif isinstance(agent, NonTabularAgent):
        v_w = agent.get_q_values_vector(state=episode_vars_list[t].state,
                weights=weights)
        v_w_next = agent.get_q_values_vector(
                state=episode_vars_list[t].state_next,
                weights=weights)

        dv_dw = agent.get_phi(episode_vars_list[t].state)

        return v_w, v_w_next, dv_dw

def get_g_array(episode_vars_list: List[EpisodeVars], gamma: float) \
        -> np.ndarray:
    g_array = np.zeros(len(episode_vars_list))

    for i in range(len(g_array))[::-1]:
        g_array[i] = episode_vars_list[i].reward

        if i != len(g_array) - 1:
            g_array[i] += gamma * g_array[i + 1]

    return g_array


'''

TODO: Really this should always be called a different thread than the GUI thread while the GUI thread has a waiting indicator.

'''
from __future__ import print_function, absolute_import
import os
import sys
import copy
import yaml
import uuid
import csv

from .test_file_extract import get_args_in_src
from .db_filestore_protocol import smb_transferer, sftp_transferer, tar_transferer

from .db_common import *
from .db_tests import test_script_obj, test_group_obj, test_group_sessions

from .db_values import value_obj_t
from .tests_group import tests_group_creator
from .db_tester import db_tester_machine

class tester_database(object):
    def __init__(self, db, sql, work_folder):
        self.db = db
        self.sql = sql
        sql.setup(db)
        self.version = sql.db_version

        self.work_folder = work_folder
        self._known_objs = {}
        self.protocol_transferers = {sftp_transferer.protocol_id : sftp_transferer(),
                                     smb_transferer.protocol_id  : smb_transferer(),
                                     tar_transferer.protocol_id : tar_transferer(self, sql, work_folder) }
        self._file_upload_cache = (None, {})
        self._new_tests_cache = (None, {})
        if not os.path.exists(work_folder):
            os.mkdir(work_folder)
        self.tester_machine = None
        self.settings = value_obj_t.get_settings_root(self)
        self.test_props = value_obj_t.get_test_props_root(self)
        self.result_values = value_obj_t.get_result_props_root(self)
        self.props_defaults = self.settings.get_child("defaults")

    def clean(self):
        for protocol_transferer in self.protocol_transferers.values():
            protocol_transferer.clean()
        self.db.clean()

    def get_filestore_protocol_id(self, protocol_name):
        cmd = self.sql.get_file_store_protocol_id(protocol_name)
        row = self.db.query_one(cmd)
        if row is None:
            return None
        return row[0]

    def add_filestore(self, host, folder, writable, protocol_id):
        host = host.upper()
        db = self.db
        sql = self.sql

        self.protocol_transferers[protocol_id].init(host, folder)

        c = db.cursor()
        found = False

        rows = c.query(sql.get_all_file_stores())
        for row in rows:
            if row[1].upper() == host:
                if row[2] == folder and row[3] == protocol_id:
                    rw = True if row[4] else False
                    writable = True if writable else False
                    if rw != writable:
                        c.update(sql.update_file_store_writable(row[0], 1 if writable else 0))
                    found = True
                    break

        if not found:
            c.insert(sql.add_file_store(host, folder, 1 if writable else 0, protocol_id ))
        db.commit()

    def add_filestore_protocol(self, name):
        db = self.db
        sql = self.sql
        c = db.cursor()

        protocol_id = c.query(sql.get_file_store_protocol_id(name))
        if protocol_id:
            return

        protocol_id = c.insert(sql.add_file_store_protocol(name))
        db.commit()

    def _new_test_group(self, db_id, name, desc, query_time=None):
        return test_group_obj(self, db_id, name, desc, query_time=query_time)

    def _new_test_obj(self, *args):
        return test_script_obj(self, *args)

    def get_all_tests(self, now=None):
        if now is None:
            now = db_ms_now()
        rows = self.db.query(self.sql.get_all_tests(now))
        return [ self._new_test_obj( row[0], row[1], row[2]) for row in rows ]

    def get_deleted_tests(self):
        now = db_ms_now()
        rows = self.db.query(self.sql.get_deleted_tests(now))
        return [ self._new_test_obj( row[0], row[1], row[2]) for row in rows ]

    def get_groups(self, now=None):
        if now is None:
            now = db_ms_now()
        rows = self.db.query(self.sql.get_groups(now))
        return [self._new_test_group(row[0], row[1], row[2]) for row in rows ]

    def get_group_by_id(self, group_id, now=None):
        if now is None:
            now = db_ms_now()
        cmd = self.sql.get_group_by_id(group_id)
        rows = self.db.query(cmd)
        if not len(rows):
            return None
        row = rows[0]
        return self._new_test_group( row[0], row[1], row[2], query_time=now)

    def get_group(self, group_name, now=None):
        if now is None:
            now = db_ms_now()
        cmd = self.sql.get_group_by_name(group_name, now)
        rows = self.db.query(cmd)
        if not len(rows):
            return None
        row = rows[0]
        return self._new_test_group(row[0], row[1], row[2])

    def _add_files(self, c, filepaths, now=None):
        file_store = c.query_one(self.sql.get_rw_file_store())
        file_store_id = file_store[0]
        file_store_host = file_store[1]
        file_store_folder = file_store[2]
        protocol_id = file_store[3]
        if protocol_id not in self.protocol_transferers:
            raise Exception("Unknown protocol for filestore.")

        protocol_transferer = self.protocol_transferers[protocol_id]

        if now is None:
            now = db_ms_now()

        tar_vstore_row = self.get_tar_virtual_filestore()

        id_cache = self._file_upload_cache[1]
        if self._file_upload_cache[0] != c:
            id_cache = {}
            self._file_upload_cache = (c, id_cache)

        fresh_files_to_upload = set()

        # Boil down to unique files that will be acturally added.
        for filepath in filepaths:
            if filepath not in id_cache:
                fresh_files_to_upload.add(filepath)

        if len(fresh_files_to_upload) > 1 and tar_vstore_row:

            filestore_protocol_transferer = protocol_transferer

            protocol_id = tar_transferer.protocol_id
            protocol_transferer = self.protocol_transferers[protocol_id]

            filename = protocol_transferer.start_tar(c)

            mod_time = db_time(time.time())
            file_size = 0

            completed_tar = filename
            completed_tar_id = c.insert(self.sql.add_file(os.path.basename(filename),
                    file_store_id, now, mod_time, file_size))

            protocol_transferer.set_tar_db_id(completed_tar_id)

            file_store_id = tar_vstore_row[0]
        else:
            protocol_transferer.open(file_store_host, file_store_folder)

        for filepath in fresh_files_to_upload:
            filename = os.path.basename(filepath)
            stat = os.stat(filepath)
            mod_time = db_time(stat.st_mtime)
            file_size = stat.st_size
            file_id = c.insert(self.sql.add_file(filename,
                file_store_id, now, mod_time, file_size))
            if not file_id:
                raise Exception("Adding file \"%s\" failed" % filename)
            protocol_transferer.upload(filepath, file_id)
            id_cache[filepath] = file_id

        if protocol_id == tar_transferer.protocol_id:
            protocol_transferer.finish_tar()
            filestore_protocol_transferer.open(file_store_host, file_store_folder)
            filestore_protocol_transferer.upload(completed_tar, completed_tar_id)
            stat = os.stat(completed_tar)
            mod_time = db_time(stat.st_mtime)
            file_size = stat.st_size
            cmd = self.sql.complete_tar_file(completed_tar_id, mod_time, file_size)
            c.update(cmd)

        # Return id array in same order as original files array.
        r = []
        for filepath in filepaths:
            file_id = id_cache.get(filepath, None)
            if file_id:
                r += [ file_id ]
        return r

    def add_files(self, filepaths, db_cursor=None, now=None):
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor
        r = self._add_files(c, filepaths, now)
        if db_cursor is None:
            db.commit()
        return r

    def get_file_to_local(self, file_id):
        if isinstance(file_id, tuple):
            file_id = file_id[-1]

        row = self.db.query_one(self.sql.get_file_filestore(file_id))
        file_store_host = row[0]
        file_store_protocol_id = row[1]
        file_store_folder = row[2]
        name = row[3]
        mod_time = row[4]
        file_size = row[5]
        local_folder = os.path.join(self.work_folder, str(file_id))
        local_file = os.path.join(local_folder, name)
        if os.path.exists(local_file):
            stat = os.stat(local_file)
            local_mod_time = db_time(stat.st_mtime)
            local_file_size = stat.st_size
            time_delta = abs(local_mod_time - mod_time)
            # Allow 1 millisecond of difference due to float/int and filesystem storage.
            if time_delta < 2 and local_file_size == file_size:
                return local_file
        elif not os.path.exists(local_folder):
            os.mkdir(local_folder)

        if file_store_protocol_id not in self.protocol_transferers:
            raise Exception("Unknown protocol for filestore.")

        protocol_transferer = self.protocol_transferers[file_store_protocol_id]

        protocol_transferer.open(file_store_host, file_store_folder)

        mod_time = db2py_time(mod_time)

        protocol_transferer.download(local_file, file_id, mod_time)

        return local_file

    def get_resource_files(self):
        rows = self.db.query(self.sql.get_resource_files())
        return dict([ row for row in rows ])

    def get_filename(self, file_id):
        row = self.db.query_one(self.sql.get_filename(file_id))
        if row is None:
            return None
        return row[0]

    def get_file_id_by_name(self, name, db_cursor=None):
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor
        row = c.query_one(self.sql.get_file_by_name(name))
        if row is None:
            return None
        return row[0]

    @staticmethod
    def _validate_args_definitions(args):
        for prop, prop_values in args.items():
            assert isinstance(prop, str), "Arg name not string"
            used = ['type', 'desc']
            prop_type = prop_values.get('type', None)
            assert prop_type is not None, "Arg has no type"
            if isinstance(prop_type, type):
                prop_pytype = prop_type
                prop_type = db_type_from_py_type(prop_type)
                assert prop_type is not None, "Arg has unknown type"
            else:
                prop_pytype = py_type_from_db_type(prop_type)
                assert prop_pytype is not None, "Arg has unknown type"
            desc = prop_values.get('desc', None)
            assert isinstance(desc, str), "Arg desc not string"

            default_value = prop_values.get('value', ValueError)
            if default_value is not ValueError:
                prop_values['default'] = prop_values.pop('value')
            else:
                default_value = prop_values.get('default', ValueError)

            if default_value is not ValueError:
                if prop_type == "float":
                    assert isinstance(default_value, float) or isinstance(default_value, int), "Arg default not of own type"
                elif prop_type == "bool":
                    assert isinstance(default_value, bool) or isinstance(default_value, int), "Arg default not of own type"
                    prop_values['default'] = bool(default_value)
                else:
                    assert isinstance(default_value, prop_pytype), "Arg default not of own type"
                used += ['default']

            if prop_type == "int" or prop_type == "float":
                for param in ["min", "max", "step"]:
                    v = prop_values.get(param, None)
                    if prop_type == "float":
                        assert isinstance(v, float) or isinstance(v, int), "Arg int/float min/max/step wrong"
                    else:
                        assert isinstance(v, prop_pytype), "Arg int/float min/max/step wrong"
                used += ["min", "max", "step"]
            used.sort()
            prop_values = list(prop_values.keys())
            prop_values.sort()
            assert used == prop_values, "Arg unknown properties"

    def add_defaults(self, args, db_cursor=None, now=None):
        self._validate_args_definitions(args)
        self.props_defaults.add_dict_tree(args, db_cursor, now)

    def get_test_by_name(self, test_name, db_cursor=None, now=None):
        if now is None:
            now = db_ms_now()
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor
        cmd = self.sql.get_test_by_name(test_name, now)
        row = c.query_one(cmd)
        if row is None:
            return None
        return self._new_test_obj( row[0], row[1], row[2])

    def get_test_by_id(self, test_id, db_cursor=None):
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor
        row = c.query_one(self.sql.get_test_by_id(test_id))
        if row is None:
            return None
        return self._new_test_obj( row[0], row[1], row[2])

    def add_test(self, local_file, db_cursor=None, now=None, test_name=None):
        if now is None:
            now = db_ms_now()
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor

        tests_id_cache = self._new_tests_cache[1]
        if self._new_tests_cache[0] != c:
            tests_id_cache = {}
            self._new_tests_cache = (c, tests_id_cache)

        test_name = test_name if test_name else os.path.basename(local_file)

        existing = self.get_test_by_name(test_name, c, now)

        if existing:
            raise Exception("Test called \"%s\" already exists." % test_name)

        test_id, file_id = tests_id_cache.get((test_name, local_file), (None, None))
        if test_id is None:
            file_id = self._add_files(c, [local_file])[0] # Look up again if need be
            test_id = c.insert(self.sql.add_test(file_id, now))
            if db_cursor is None:
                db.commit()
            tests_id_cache[(test_name, local_file)] = (test_id, file_id)
        return self._new_test_obj( test_id, test_name, file_id)

    def _update_groups_with_new_test(self, groups, old_test, new_test, db_cursor=None, now=None):
        for group in groups:
            for n in range(0, len(group.tests)):
                test = group.tests[n]
                if test.id == old_test.id:
                    group.tests[n] = new_test
                    new_test.pending_properties = \
                        test.pending_properties
            group.updated_db(db_cursor, now)

    def _get_default_args(self, local_folder, db_cursor=None, now=None):
        r = {"exit_on_fail": { 'desc'   : 'Exit on Fail',
                               'type'   : 'bool',
                               'default': True }}
        defaults_file = os.path.join(local_folder, "args.yaml")
        org = self.props_defaults.get_as_dict_tree(db_cursor, now)
        r.update(org)
        if os.path.exists(defaults_file):
            with open(defaults_file) as f:
                defaults_gen=yaml.safe_load_all(f)
                new_args = [root for root in defaults_gen][0]
                for arg, arg_body in new_args.items():
                    old_arg = org.get(arg, None)
                    if old_arg:
                        # We can let something things change, but not type.
                        assert old_arg['type'] == arg_body['type'], "Type change of default arg."
                r.update(new_args)
        return r

    def group_from_dict(self, group_data, folder, db_cursor=None, now=None):
        if now is None:
            now = db_ms_now()
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor

        default_args = self._get_default_args(folder, c, now)
        tests = []
        for test_name, test_args in group_data["tests"]:
            if 'exit_on_fail' not in test_args:
                test_args['exit_on_fail'] = True

            if isinstance(test_name, list):
                assert len(test_name) == 2, "If list is given for testname, it should be name then filename. : " + str(testname)
                filename = test_name[1]
                test_name = test_name[0]
                test_obj = self.get_test_by_name(filename, c, now)
                test_filename = os.path.join(folder, filename)
                if test_obj:
                    test_obj = copy.copy(test_obj)
                    test_obj.name = test_name
            else:
                test_obj = self.get_test_by_name(test_name, c, now)
                test_filename = os.path.join(folder, test_name)
            if not test_obj:
                if not os.path.exists(test_filename):
                    raise Exception("Test file '%s' not found" % test_filename)
                file_args = get_args_in_src(test_filename)
                for arg in file_args:
                    if arg not in test_args:
                        arg_def = default_args[arg]
                        default_val = arg_def.get("default", None)
                        if default_val is not None:
                            test_args[arg] = default_val
                        else:
                            print('WARNING importing test "%s", argument "%s", no value at load time.' % (test_name, arg))
                test_obj = self.add_test(test_filename, c, now, test_name)
            for arg_key in test_args:
                arg_details = default_args[arg_key]
                if arg_details['type'] == 'file':
                    filename = test_args[arg_key]
                    if isinstance(filename, str):
                        file_id = self.get_file_id_by_name(filename, c)
                        if file_id is None:
                            filename = os.path.abspath(os.path.join(folder, filename))
                            file_id = self._add_files(c, [filename])[0]
                        test_args[arg_key] = (dbfile, filename, file_id)
            test_obj.pending_properties = test_args
            tests += [test_obj]

        test_used = {}
        for test in tests:
            if test.name in test_used:
                count = test_used[test.name]
                test.name += "#%02i" % count
                test_used[test.name] = count + 1
            else:
                test_used[test.name] = 1

        r = self.add_group(group_data["name"],
                           group_data["desc"],
                           tests, c, now, group_data.get("note", None))

        self.add_defaults(default_args, c, now)

        if db_cursor is None:
            db.commit()

        return r

    def load_groups(self, filename, db_cursor=None, now=None):
        db = self.db
        if db_cursor:
            c = db_cursor
        else:
            c = db.cursor()
        if not now:
            now = db_ms_now()
        r = []
        folder = os.path.dirname(filename)
        with open(filename) as f:
            root_def_gen=yaml.safe_load_all(f)
            root_def = [root for root in root_def_gen][0]

            templates = root_def.get('templates', {})
            groups_list = root_def['groups']

            for group_data in groups_list:
                template_name = group_data.get('template', None)
                if template_name:
                    template_tests = templates.get(template_name, None)
                    if not template_tests:
                        raise Exception("Template '%s' used but not found." % template_name)

                    group_tests = group_data.get('tests', [])

                    for n in range(0, len(template_tests)):
                        template_test = template_tests[n]
                        if n == len(group_tests):
                            group_tests += [ copy.copy(template_test) ]
                        else:
                            group_test = group_tests[n]
                            if not len(group_test):
                                group_tests[n] = copy.copy(template_test)

                    group_data['tests'] = group_tests

                r += [self.group_from_dict(group_data, folder, c, now)]

            if not db_cursor:
                db.commit()
        return r

    def get_db_now():
        return None

    def add_group(self, name, desc, tests, db_cursor=None, now=None, note=None):
        if now is None:
            now = db_ms_now()
        db = self.db
        if db_cursor is None:
            c = db.cursor()
        else:
            c = db_cursor
        group_id = c.insert(self.sql.add_test_group(name, desc, now, note))

        group = self._new_test_group( group_id, name, desc)

        order_pos = 0
        for test in tests:
            group.add_test(test, c, order_pos, now)
            order_pos += 1

        if db_cursor is None:
            db.commit()

        return group

    def get_sessions(self, session_ids):
        assert len(session_ids)
        cmd = self.sql.get_sessions(session_ids)
        r = []
        for row in self.db.query(cmd):
            group = self._new_test_group( row[2], row[3], row[4])
            r += [ test_group_sessions(group, self, row[0], row[1], row[5], row[6]) ]
        return r

    def load_filestores(self, db_def):

        file_stores = db_def['file_stores']

        smbs = file_stores.get("smb", None)
        if smbs:
            for filestore in smbs:
                self.add_filestore(filestore[0], filestore[1], True,
                                   smb_transferer.protocol_id)

        sftps = file_stores.get("sftp", None)
        if sftps:
            for filestore in sftps:
                self.add_filestore(filestore[0], filestore[1], True,
                                   sftp_transferer.protocol_id)

    def get_machine(self, machine_id):
        return db_tester_machine.get_by_id(self, machine_id)

    def get_own_machine(self):
        if not self.tester_machine:
            self.tester_machine = db_tester_machine.get_own_machine(self)
        return self.tester_machine

    def get_dev(self, dev_uuid):
        raise NotImplementedError

    def get_dev_by_sn(self, serial_number):
        raise NotImplementedError

    def get_tar_virtual_filestore(self):
        db = self.db
        sql = self.sql
        c = db.cursor()
        return c.query_one(self.sql.get_tar_virtual_filestore())

    def generate_csv(self, outfile, before=None, after=None):
        db = self.db
        sql = self.sql
        c = db.cursor()
        csv_results = c.query(sql.get_csv_results(before, after))
        try:
            with open(outfile, 'w', newline='', encoding='utf-8') as csv_file:
                csv_writer = csv.writer(csv_file, dialect='excel')
                csv_writer.writerow(("Test Group", "Serial Number", "Result", "Timestamp"))
                for i in csv_results:
                    row = list(i)
                    row[2] = "PASS" if row[2] else "FAIL"
                    csv_writer.writerow(row)
        except Exception as e:
            print(f"Error: couldn't open file {outfile}:\n{e}", flush=True)
            return False
        return True


#!/usr/bin/env python
#
# Poll arduino for latest weather data, and insert in to RRD.
# Depends on having an RRD precreated. Refer to scripts in
# weather package.

import datetime, time, rrdtool, serial, sys

# Serial device properties
arduinodev = "/dev/weatherstation"
arduinospeed = 115200
# Path to our RRD
rrdfile = "/home/peter/weather/weather.rrd"
# Path to our log file
logfile = "/home/peter/weather/weather.log"

# Retrieve sensor data
conn = serial.Serial(arduinodev, arduinospeed)
conn.write("get;")
datain = conn.readline()
conn.close()

# Sensor data is received in the form
# "<channel number>,<temperature>,<humidity>,<time offset>;\n"
# First we split this out to a list of values,
# then cast each to numeric values and assign to variables.
x = datain.split(";")[0].split(",")
if x[0] is "1":
    temperature = float(x[1])
    humidity = int(x[2])
    timeoffset = int(x[3])
    print("Received %s;%s;%s" % (temperature, humidity, timeoffset))
else:
    print("Unknown channel received.")
    sys.exit(1)

# Figure out the time the recording was taken.
polltime = long(time.time() - timeoffset)

# Write values to RRD
rrdtool.update(rrdfile,
               "--template", "temperature:humidity",
               "%s:%s:%s" % (polltime, temperature, humidity))
# Write values to logfile
now = datetime.datetime.now()
lfh = open(logfile, 'a')
lfh.write("%s %s:%s:%s\n" % (now.strftime("%b %d %H:%M:%S"), polltime,
                           temperature, humidity))
lfh.close()


import mistune

from django import forms

from .models import Comment

class CommentForm(forms.ModelForm):
    nickname = forms.CharField(
        label='昵称',
        max_length=10,
        widget=forms.widgets.Input(
            attrs={'style':"background-color: #ffffff;border-radius: 40px;font-size:15px; border:1px solid #BFBFBF;height: 30px;padding-left: 10px; ",'placeholder':'"少侠！请留下姓名！"' }
        )
    )

    content = forms.CharField(
        label="内容",
        max_length=500,
        widget=forms.widgets.Textarea(
            attrs={'style':"background-color: #ffffff;border-radius: 10px;border:1px solid #BFBFBF;min-height: 100px;width: 100%;padding-left: 10px;" }
        )
    )

    def clean_content(self):
        content = self.cleaned_data.get('content')
        if len(content) < 10:
            raise forms.ValidationError('内容长度怎么能这么短呢！！')
        content = mistune.markdown(content)
        return content

    class Meta:
        model = Comment
        fields = ['nickname', 'content']


from parsers.functions.get_content import get_content

def parse_one_punch(url):
    content = get_content(url)

    full_names = list()
    images = list()
    tables = content.find('div', {'class', 'wds-tab__content wds-is-current'}).find_all(
        'table')
    for table in tables:
        inner_tables = table.find_all('table')
        for inner_table in inner_tables:
            div = inner_table.find('div').find('div', {'class': 'floatnone'})
            a = div.find('a')
            full_name = a.get('title')
            full_names.append(full_name)
            image = a.find('img').get('src')
            images.append(image)
    return {'full_name': full_names, 'image': images}

one_punch_url = 'https://onepunchman.fandom.com/wiki/Characters'
one_punch_data = parse_one_punch(one_punch_url)
print(one_punch_data)
# df = pd.DataFrame(one_punch_data)
# print(df)


# Given an array of strings strs, group the anagrams together. You can return the answer in any order.

# An Anagram is a word or phrase formed by rearranging the letters of a different word or phrase, typically using all the original letters exactly once.

# Example 1:

# Input: strs = ["eat", "tea", "tan", "ate", "nat", "bat"]
# Output: [["bat"], ["nat", "tan"], ["ate", "eat", "tea"]]

import collections

class Solution:
    def groupAnagrams(self, strs: list[str]) -> list[list[str]]:
        ans = collections.defaultdict(list)

        for a in strs:
            count = [0] * 26

            for s in a:
                count[ord(s) - ord('a')] += 1

            ans[tuple(count)].append(a)

        return ans.values()