# Author: Anthony Kai Kwang Ma
# Email: anthonyma27@gmail.com, akma327@stanford.edu
# Date: 05/14/17
# gpcr_atomic_contact_fingerprint.py

import os
import sys
import json
import csv
import numpy as np
import re
from utils import *

USAGE_STR = """

# Purpose
# Input: GPCR-Contacts at atomic resolution for a set of receptors.
# Output: Fingerprint Flareplot format json for visualization.

# Usage
# python gpcr_contact_fingerprint.py <OUTPUT_DIR> <INPUT_FILE_1> <INPUT_FILE_2> ...

# Arguments
# <OUTPUT_DIR> Output directory for <interaction_type>.json
# <INPUT_DIR_1> Input contacts table.txt file for a particular receptor
# ...
# <INPUT_DIR_N>

# Example
OUTPUT_DIR="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-fingerprint/opioid"
INPUT_FILE_1="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRM_MOUSE_4DKL_A/OPRM_MOUSE_4DKL_A_table.txt"
INPUT_FILE_2="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRD_HUMAN_4N6H_A/OPRD_HUMAN_4N6H_A_table.txt"
INPUT_FILE_3="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRK_HUMAN_4DJH_A/OPRK_HUMAN_4DJH_A_table.txt"
INPUT_FILE_4="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRX_HUMAN_4EA3_A/OPRX_HUMAN_4EA3_A_table.txt"
cd /scratch/PI/rondror/akma327/GPCRContacts/src/fingerprint
python gpcr_atomic_contact_fingerprint.py $OUTPUT_DIR $INPUT_FILE_1 $INPUT_FILE_2 $INPUT_FILE_3 $INPUT_FILE_4

OUTPUT_DIR="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-fingerprint/muscarinic"
INPUT_FILE_1="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM2_HUMAN_3UON_A/ACM2_HUMAN_3UON_A_table.txt"
INPUT_FILE_2="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM3_RAT_4DAJ_A/ACM3_RAT_4DAJ_A_table.txt"
INPUT_FILE_3="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM1_HUMAN_5CXV_A/ACM1_HUMAN_5CXV_A_table.txt"
INPUT_FILE_4="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM4_HUMAN_5DSG_A/ACM4_HUMAN_5DSG_A_table.txt"
cd /scratch/PI/rondror/akma327/GPCRContacts/src/fingerprint
python gpcr_atomic_contact_fingerprint.py $OUTPUT_DIR $INPUT_FILE_1 $INPUT_FILE_2 $INPUT_FILE_3 $INPUT_FILE_4

"""

K_MIN_ARG = 3

def sort_paths(INPUT_FILES):
	path_comp = [p.split("/") for p in INPUT_FILES]
	sorted_input_files = ["/".join(sp) for sp in sorted(path_comp, key=lambda x: x[-1])]
	labels = [sp[-1].strip("_table.txt") for sp in sorted(path_comp, key = lambda x: x[-1])]
	return sorted_input_files, labels

def get_edge_and_frameDict(sorted_input_files, labels):
	"""
		sorted_input_files: Tables containing the gpcrdb pairs and atomic resolution interaction data
		labels: GPCR Receptor name
	"""
	frameDict = {}
	edge_dict = {}

	# residue_to_atomic_contacts = {itype: {gpcrdb1:gpcrdb2: {receptor_index1: [list of atomic resolution contacts], ...}}}
	# {"hbbb": {3x53:3x54: {0:[ALA168-N:VAL169-N, ALA168-O:VAL169-N,...], 1: []}}}
	residue_to_atomic_contacts = {}
	for i in range(len(sorted_input_files)):
		p, label = sorted_input_files[i], labels[i]
		frameDict[i] = label

		f = open(p, 'r')
		for line in f:
			gpcrdb1, gpcrdb2, atom_list, itype = line.strip().split("\t")
			if(itype not in edge_dict):
				edge_dict[itype] = {}
			if((gpcrdb1, gpcrdb2) not in edge_dict[itype]):
				edge_dict[itype][(gpcrdb1, gpcrdb2)] = set()
			edge_dict[itype][(gpcrdb1, gpcrdb2)].add(i)

			### Atomic level contacts encoding into flareplot json
			gpcrdb_pair = gpcrdb1 + ":" + gpcrdb2
			if(itype not in residue_to_atomic_contacts):
				residue_to_atomic_contacts[itype] = {}
			if(gpcrdb_pair not in residue_to_atomic_contacts[itype]):
				residue_to_atomic_contacts[itype][gpcrdb_pair] = {}
			if(atom_list not in residue_to_atomic_contacts[itype][gpcrdb_pair]):
				residue_to_atomic_contacts[itype][gpcrdb_pair][i] = []
			residue_to_atomic_contacts[itype][gpcrdb_pair][i].append(atom_list)

	return edge_dict, frameDict, residue_to_atomic_contacts

def fingerprint(OUTPUT_DIR, INPUT_FILES):
	if not os.path.exists(OUTPUT_DIR):
		os.makedirs(OUTPUT_DIR)

	sorted_input_files, labels = sort_paths(INPUT_FILES)
	edge_dict, frameDict, residue_to_atomic_contacts = get_edge_and_frameDict(sorted_input_files, labels)
	for itype in edge_dict:
		out_path = OUTPUT_DIR + "/" + itype + ".json"
		edges = []
		for edge in edge_dict[itype]:
			name1, name2 = edge
			frames = sorted(list(edge_dict[itype][edge]))
			edges.append({"name1": name1, "name2": name2, "frames": frames})

		json_dict = partial_input
		json_dict["edges"] = edges
		json_dict["frameDict"] = frameDict

		### Atomic level contacts encoding into flareplot json
		json_dict['residue_to_atomic_contacts'] = residue_to_atomic_contacts[itype]

		with open(out_path, 'w') as outfile:
			json.dump(json_dict, outfile)

if __name__ == "__main__":
	if(len(sys.argv) < K_MIN_ARG):
		print(USAGE_STR)
		exit(0)
	OUTPUT_DIR = sys.argv[1]
	INPUT_FILES = sys.argv[2:]
	fingerprint(OUTPUT_DIR, INPUT_FILES)


# Создайте словарь со списком вещей для похода в качестве ключа и их массой в качестве значения.
# Определите какие вещи влезут в рюкзак передав его максимальную грузоподъёмность.
# Достаточно вернуть один допустимый вариант.
# *Верните все возможные варианты комплектации рюкзака. -

max_weight = 10

item_dict = {'палатка': 5, 'спальник': 0.8, 'посуда': 1, 'горелка': 1, 'фонарь': 0.3, 'батарейки': 0.4, 'дождевик': 0.5,
             'тушенка': 2, 'макароны': 2, 'спиннинг': 1.1, 'пуховик': 0.5, 'одежда': 2, 'нож': 0.2, 'сахар': 0.15,
             'соль': 0.1, 'фотоаппарат': 2, 'бумага': 0.15}

package = []
total_weight = 0
for item, weight in item_dict.items():
    if (total_weight + weight) <= max_weight and weight <= max_weight:
        package.append(item)
        total_weight += weight
print(f'В рюкзак с грузоподъёмностью {max_weight} кг влезут:', ', '.join(package), '\nобщим весом', total_weight, 'кг')


from functions.input_functions import (yes_no_input, table_input,
                                       collect_query_params,
                                       conjoin_filter)
from functions.db_functions import (get_conn, get_curs, disconnect,
                                    filter_table)

conn = get_conn()
curs = get_curs(conn = conn)

if __name__ == '__main__':

    querying = True
    while querying:
        print('Enter exit to abandon statistics')
        table = table_input()
        if table:
            params = collect_query_params(table = table)
            if params:
                params = conjoin_filter(params)
                new = filter_table(curs = curs, table = table, fields = params)
                print('Columns: \n' + ' '.join(d.name for d in new[0]))
                print(new[1][:30])
            print('Filter another table?')
            querying = yes_no_input()
        else:
            querying = False

    disconnect(conn = conn, curs = curs)


import pandas as pd
from datetime import datetime
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import LinearRegression

stock = pd.read_csv("sphist.csv")
stock["Date"] = pd.to_datetime(stock["Date"])
boolean = stock["Date"] > datetime(year=2015, month=4, day=1)
stock = stock.sort_values(by="Date", ascending=True)
# print(stock.columns)
stock["day_5_price"] = stock.rolling(window=5,
                                     on="Date").mean().loc[:,"Close"]
stock["day_30_price"] = stock.rolling(window=30,
                                      on="Date").mean().loc[:,"Close"]
stock["day_365_price"] = stock.rolling(window=365,
                                       on="Date").mean().loc[:,"Close"]
stock["mean_ratio"] = stock["day_5_price"] / stock["day_365_price"]
stock["day_5_std"] = stock.rolling(window=5,
                                   on="Date").std().loc[:,"Close"]
stock["day_365_std"] = stock.rolling(window=5,
                                     on="Date").std().loc[:,"Close"]
stock["std_ratio"] = stock["day_5_std"] / stock["day_365_std"]

# print(stock["day_5"].head(7))
stock = stock.shift(periods=1)
# print(stock.head(7))
stock = stock[stock["Date"] > datetime(year=1951, month=1, day=3)]
stock = stock.dropna(axis=0).copy()
# print(stock['Close'].isnull().sum())
train = stock[stock["Date"] < datetime(year=2013, month=1, day=1)]
test = stock[stock["Date"] >= datetime(year=2013, month=1, day=1)]
# print(len(train), len(test))
model = LinearRegression()
features = train.drop(['Close', 'High', 'Low', 'Open', 'Volume',
                    'Adj Close', 'Date'], axis=1).copy()
target = train["Close"]
model.fit(features, target)
test_features = test.drop(['Close', 'High', 'Low', 'Open', 'Volume',
                    'Adj Close', 'Date'], axis=1).copy()
predictions = model.predict(test_features)
mae = mean_absolute_error(test["Close"], predictions)
print(len(test['Close']))
print(len(predictions))
print(test.columns)


import numpy as np
import matplotlib.pyplot as plt

cpu = []
gpu = []
with open('cpu_therm') as f:
	for line in f:
		data = line
		cpu.append(float(data))

with open('gpu_therm') as f:
	for line in f:
		data = line
		gpu.append(float(data))

line_cpu, = plt.plot(cpu, label='CPU')
line_gpu, = plt.plot(gpu, label='GPU')
plt.ylabel('temperature')
plt.legend(handles=[line_cpu, line_gpu])

plt.show()


# Ask the user for a number.
# Depending on whether the number is even or odd, print out an appropriate message to the user.
# Hint: how does an even / odd number react differently when divided by 2?

def ask_number():
    number=int(input("Enter the number: "))
    return number

def check_number(num):
    check=num % 2
    return check

num=ask_number()
result=check_number(num)
if result is 0:
    print("The number is devided by 2")
else:
    print("The number is not devided by 2")

var = True.__doc__


def gray_code(n):
    if not n:
        return [0]

    result = gray_code(n-1)
    num = 1 << n-1

    for i in range(len(result)-1, -1, -1):
        result.append(num+result[i])

    print(result)
    return result

gray_code(3)


import wikipedia
from collections import Counter
from nltk import word_tokenize
from nltk import sent_tokenize
from nltk import pos_tag
from nltk.corpus import wordnet
from nltk.wsd import lesk

def noun_finder(content):
    '''
    This function finds all the nouns from a Wikipedia page and returns them
    in a list along with the corresponding sentence.
    '''
    sentence_list = []
    noun_list = []
    sent_tokens = sent_tokenize(content)
    for elem in sent_tokens:
        sentence_list.append(word_tokenize(elem))
    for sent in sentence_list:
        tagged_sent = pos_tag(sent)
        for word in tagged_sent:
            if word[1][:2] == 'NN':
                noun_list.append([word[0], sent])
    return noun_list

def synsets(noun_list):
    '''
    Finds all corresponding synsets and their senses to the nouns, tracks the amount of nouns and
    whether they are unisemous or polysemous.
    '''
    noun_count = 0
    ps_words = 0
    us_words = 0
    total_ps_senses = 0
    words_10 = 0
    nouns_list = []
    full_nouns_list = []
    synset_list, sense_list, diff_senses = [], [], []
    for noun in noun_list:
        synsets = wordnet.synsets(noun[0], 'n')
        synset_list.append(synsets)
        if len(synsets) == 1:
            sense_list.append(len(synsets))
            noun_count += 1
            us_words += 1
        elif len(synsets) > 1:
            sense_list.append(len(synsets))
            noun_count += 1
            total_ps_senses += len(synsets)
            ps_words += 1
            if words_10 < 10:
                nouns_list.append(noun)
                words_10 += 1
            if words_10 != 'loop':
                full_nouns_list.append(noun)
    return synset_list, nouns_list, noun_count, us_words, ps_words, total_ps_senses, sense_list, full_nouns_list

def count_nouns(full_nouns_list):
    noun_definitions = {}
    for noun in full_nouns_list:
        if noun[0] in noun_definitions.keys():
            noun_definitions[noun[0]].append(noun[1])
        else:
            noun_definitions[noun[0]] = [noun[1]]
    for key, values in noun_definitions.items():
        if len(values) > 1:
            for value in values:
                print(key, " : ", lesk(value, key).definition())
            print("\n")

def main():
    wiki = wikipedia.page('Colonel Sanders')
    content = wiki.content
    noun_list = noun_finder(content)
    synset_list, first_nouns, noun_count, us_words, ps_words, total_ps_senses, sense_list, full_nouns_list = synsets(noun_list)
    print('1.\nThere were a total of {0} nouns in the text. {1} of those were unisemous and {2} were polysemous. '\
          'The proportion of polysemous nouns is {3:.2f}.\n'.format(noun_count, us_words, ps_words, ps_words/noun_count))
    if ps_words == 1:
        print('2.\nThis page contains a polysemous word.\n')
    elif ps_words == 0:
        print('2.\nThis page does not contain any polysemous words.\n')
    elif ps_words > 1:
        print('2.\nThis page contains multiple polysemous words.\n')
    print('3.\nThe average number of senses per polysemous words is {0:.2f}.\n'.format(total_ps_senses/ps_words))
    print('4.\nCount for different number of senses (format: nr of senses:count):\n')
    print(Counter(sense_list))
    print('\n5.')
    for noun in first_nouns:
        print(noun[0], " : ", lesk(noun[1], noun[0]).definition())
    print("\nIn this case it get about 2/3 words right. Especially words that are very context specific like chain and David"\
    " seem to be difficult, however it does get some tricky cases right like the word chicken. In this case it was"\
    " indeed meant as the food chicken and not the animal itself.\n")
    print("6.\nNo, this did not happen in question 5. However, it can happen, depending on the sentence that belongs to the word. If the word around the main word"\
          " differs too much in context, the wrong definition will be found. If you look at the same words over the entire Wikipedia page,"\
          " there are a lot of words that do differ in definition. The function count_nouns can be used to look at all the nouns that have"\
          " mulitple occurances and their definitions. Some of these definitions differ.\n")
    print("7.\nI did expect that some words would be correct, when they were not. For instance 'David' is placed in the correct context"\
          " and was still given a wrong definition.\n")
    print("8.\nDisambiguation can help assign the right meaning of a word to that word. This can help link to the right Wikipedia"\
    " page so that for example the sentence \"I am going to school\" will link to the institute and not to the page related to fish.")

if __name__ == '__main__':
    main()


z = str(input())
t = 0
prev = ord('a')
for i in z:
        diff=abs(ord(i)-prev)

        if diff>13:
            t+=26-diff
        else:
            t+=diff
        prev=ord(i)

print(t)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 15 10:23:37 2019

@author: thomas
"""

import numpy as np
import matplotlib.pyplot as plt

# function to create the uniform distribution
def uniform(x,a,b):

    f=(x>=a) * (x<=b)
    f=1.0/(b-a)*f
    return(f)

def gauss(x,mu,s):

    f=1.0/np.sqrt(2*np.pi)/s*np.exp(- np.power(x-mu,2)/2/np.power(s,2) )
    return(f)

x = np.arange(-5, 6, 0.01)
a=-2.0
b=+2.0
mu=1.0
s=2.0

fg=gauss(x,mu,s)
fs=uniform(x,a,b)

font = {'family' : 'serif',
        'weight' : 'normal',
        'size'   : 16}
plt.rc('font', **font)

# Plot in frequency domain
plt.figure(1)
plt.xlabel('$x$',**font)
plt.ylabel('Distribution')
plt.plot(x,abs(fg),label='Gaussian')
plt.plot(x,abs(fs),label='uniform')
plt.xlim(-5.0,6.0)
plt.legend(bbox_to_anchor=(1.3, 1.05))
plt.tight_layout()
plt.savefig('pdfs.png')


import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from time import sleep
import random
import datetime #for converting sales dates

class SpidermanSpider(CrawlSpider):
    name = 'spiderman'
    allowed_domains = ['ebay.com']
    # start_urls = ['']
    user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'

    def start_requests(self):
        yield scrapy.Request(url="https://www.ebay.com/sch/259104/i.html?_from=R40&_nkw=spiderman&LH_Sold=1&LH_Complete=1",
        headers={
            'User-Agent': self.user_agent
        })

    rules = (
        Rule(LinkExtractor(restrict_xpaths="//ul[@class='srp-results srp-list clearfix']/li[contains(@class, 's-item s-item__pl-on-bottom')]/div/div/a"),
        callback='parse_item', follow=True, process_request='set_user_agent'),
        Rule(LinkExtractor(restrict_xpaths="//a[@class='pagination__next icon-link']"), process_request='set_user_agent'),
    )

    def convert_to_int(self, value):

        if value == None: #this works
            # print(value)
            return value

        else:
            value=int(value) #turn value from string to int
            return value

    def split_grade(self, value):

        if value == None: #this works
            # print(value)
            return value

        else:
            float_strings = value.split() #splits the grade entry on the spaces
            for f in float_strings:
                f= f.strip("+-") #if there are + or - on the grade number, this will get rid of them
                try:
                    f=float(f) #turn f into a float
                    return f
                except ValueError: #this is where parts of the the string that are not the grade number go to die
                    pass

    def convert_year(self, value):

        if value == None: #this works
            # print(value)
            return value

        else:
            try:
                format = "%Y"

                # convert from string format to datetime format
                theDay = datetime.datetime.strptime(value, format)
                return theDay.year
                # print(theDay.year)

            except ValueError: #this is where parts of the the string that are not the a 4 digit year go to die. Sometimes seller's put in ranges
                pass

    def verify_issue(self, value):

        if value == None: #if nothing is entered in issue
            #print(None)
            return value

        elif value.isdecimal(): #will only work if there is a single number, no commas or letters
            #print(value)
            return value

        elif "N" in value[:1] or "n" in value[:1]: #will work if the seller wrote something that starts with the letter "n" like "no, num, no." letter in the issue section
            if "," not in value and "-" not in value and "&" not in value and "/" not in value: #if these are NOT present, would indicate NOT a lot is being listed with more than one issue
                result = ''
                for char in value:
                    if char in '1234567890':
                        result += char
                #print(result)
                return result

            else: # if the above symbols are present, that means a lot is probably listed
                #print(8888)
                return "9999"

        # elif value.isalpha(): #will work if the seller wrote something that starts with a letter in the issue section like "various" or "see description"
        #     #print("is alpha")
        #     return None

        elif "," in value or "-" in value or "&" in value or " " in value or "/" in value: #this means a lot is being listed with more than one issue
            #print(9999)
            return "9999"
        else: #this will strip out any letters or symbols
            result = ''
            for char in value:
                if char in '1234567890':
                    result += char
            if result == "":
                return None
            else:
            # print(result)
                return result

    def id_variant(self, value):
        if value == None: #if nothing is entered in issue
            #print(None)
            return value
        else:
            result = ''
            for char in value:
                if char not in '1234567890#-,/':
                    result += char
            if result == "":
                return None
            else:
            # print(result)
                return result

    def split_currency(self, value):
        substring1 = "$"
        free = "Free"

        if value == None or value == free:
            value== 'US'
            return value

        elif substring1 in value:
            return value.split("$")[0].strip('\n\t\t\t\t\t\t\t\t\t\t') # split gets currency, strip removes sales price characters

        else:
             return value.split()[0]

    def split_price(self, value):
        substring1 = "$"
        #free = "Free"
        if value != None and substring1 in value:

            value = value.split("$")[1].strip('\n\t\t\t\t\t\t\t\t\t\t').replace(",", "") # split gets currency, strip removes sales price characters
            value = float(value)
            value = format(value, '.2f')
            return value
        elif value == None or value == 'free' or value == 'Free' or value == 'FREE':
            value = float('0.00')
            return value
        else:
            value = value.split()[1].replace(",", "")
            value = float(value)
            value = format(value, '.2f')
            return value

    # def remove_Best_Offer_characters(self, value):
    #     if value is not None:
    #         return 1
    #     else:
    #         return 0

    def remove_time_of_sale_characters(self, value):
        if value is not None:
            value = value.strip('\r\n\t\t\t')
            # format
            format = "%b %d, %Y"

            # convert from string format to datetime format
            theDay = datetime.datetime.strptime(value, format)

            # get the date from the datetime using date()
            # function
            return theDay.date()
        else:
            return value

    def sell_method(self, value):
        highest_bidder= 'class="notranslate vi-VR-cvipPrice"'
        best_offer_accepted='style="font-weight:normal;text-decoration:line-through"'
        if value != None and highest_bidder in value:
            return "highest_bidder"
        elif value != None and best_offer_accepted in value:
            return "best_offer_accepted"
        elif value == None:
            return value
        else:
            return "buy_it_now"

    def truncate_string(self, value):
        value = value[:2000]
        return value

    def set_user_agent(self, request, spider):
        request.headers['User-Agent'] = self.user_agent
        return request

    def parse_item(self, response):
        sleep(random.uniform(0,1))
        yield {
            'Title': response.xpath("//h1[@id='itemTitle']/text()").get(), #complete
            'Series_Title': response.xpath("//span[contains(string(), 'Series Title:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Issue_Number': self.verify_issue(response.xpath("//span[contains(string(), 'Issue Number:')]/../../../following-sibling::div[1]/div/div/span/text()").get()),
            'Issue_Variant': self.id_variant(response.xpath("//span[contains(string(), 'Issue Number:')]/../../../following-sibling::div[1]/div/div/span/text()").get()),
            'Image_url': response.xpath("//img[@id='icImg']/@src").get(), #complete
            'Condition': response.xpath("//div[@id='vi-itm-cond']/text()").get(), #complete
            'Grade': self.split_grade(response.xpath("//span[contains(string(), 'Grade:')]/../../../following-sibling::div[1]/div/div/span/text()").get()), #complete
            'Publisher': response.xpath("//span[contains(string(), 'Publisher:')]/../../../following-sibling::div[1]/div/div/span/text()").get(), #the [2] at the end may not be necessary?
            'Publication_Year': self.convert_year(response.xpath("//span[contains(string(), 'Publication Year:')]/../../../following-sibling::div[1]/div/div/span/text()").get()), #the [2] at the end may not be necessary?
            'Format': response.xpath("//span[contains(string(), 'Format:')]/../../../following-sibling::div[1]/div/div/span/text()").get(), #the [2] at the end may not be necessary?
            'Tradition': response.xpath("//span[contains(string(), 'Tradition:')]/../../../following-sibling::div[1]/div/div/span/text()").get(), #the [2] at the end may not be necessary?
            'Features': response.xpath("//span[contains(string(), 'Features:')]/../../../following-sibling::div[1]/div/div/span/text()").get(), #the [2] at the end may not be necessary?
            'Era': response.xpath("//span[contains(string(), 'Era:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Type': response.xpath("//span[contains(string(), 'Type:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Genre': response.xpath("//span[contains(string(), 'Genre:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Character': self.truncate_string(response.xpath("//span[contains(string(), 'Character:')]/../../../following-sibling::div[1]/div/div/span/text()")).get(),
            'Series': response.xpath("//span[contains(string(), 'Series:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Universe': response.xpath("//span[contains(string(), 'Universe:')]/../../../following-sibling::div[1]/div/div/span/text()").get(),
            'Sale_Currency': self.split_currency(response.xpath("//span[contains(@class, 'notranslate')and not(contains(@id,'fshippingCost'))]/text()").get()), #complete
            'Sale_Price': self.split_price(response.xpath("//span[contains(@class, 'notranslate')and not(contains(@id,'fshippingCost'))]/text()").get()), #complete
            'Shipping_Price': self.split_price(response.xpath("//span[@id='fshippingCost']/span/text()").get()), #complete
            #'combined_sale_shipping_price': Sale_Price.value+Shipping_Price
            'Sell_Method': self.sell_method(response.xpath("//span[contains(@class, 'notranslate')and not(contains(@id,'fshippingCost'))]").get()), #complete
            'Num_of_Bids': self.convert_to_int(response.xpath("//a[@id='vi-VR-bid-lnk-']/span[1]/text()").get()), #complete
            # 'Shipping_Currency': self.split_currency(response.xpath("//span[@id='fshippingCost']/span/text()").get()), #complete
            'Time_of_Sale': self.remove_time_of_sale_characters(response.xpath("//div[@class='u-flL  vi-bboxrev-posabs vi-bboxrev-dsplinline ']/span/text()").get()), #try getall to get the date and the time
            'Sellers_Location': response.xpath("//div[@class='u-flL lable' and contains(text(),'Located in:')]/following-sibling::div/text()").get(), #complete
            'Sellers_Handle': response.xpath("//span[@class='mbg-nw']/text()").get(), #complete
            'Sellers_Feedback_Score': self.convert_to_int(response.xpath("//a[contains(@aria-label, '(feedback score)')]/text()").get()),  #complete
            'listing_url': response.url
       }


from intermediateApi import ffi, lib

from c_types import SizeT, Void
from ua_types_parent import _ptr, _val, _is_null, _get_c_type, _is_ptr, UaType
from typing import Union, List

from ua_types_primitive import UaInt64, UaDouble, UaBoolean, UaString

class UaList(UaType):
    def __init__(self, val: Union[UaType, List] = None, size=None, ua_class=None, list_of_pointers=False):
        # TODO: implement default for list_of_pointers (e.g. primitives, NodesIds...)
        self._list_of_pointers = list_of_pointers
        c_type = ""
        if type(val) is list:
            if ua_class is None:
                if type(val[0]) is int:
                    c_type = "UA_Int64"
                    ua_class = UaInt64
                elif type(val[0]) is float:
                    c_type = "UA_Double"
                    ua_class = UaDouble
                elif type(val[0]) is str:
                    c_type = "UA_String"
                    ua_class = UaString
                    val = list(map(lambda string:
                                   UaString(string)._ptr if self._list_of_pointers else UaString(string)._val, val))
                elif type(val[0]) is bool:
                    c_type = "UA_Boolean"
                    ua_class = UaBoolean
            elif isinstance(ua_class(), UaType):
                c_type = _get_c_type(ua_class()._ptr)
            else:
                raise AttributeError("'ua_class' has to be None or a Subclass of UaType.")

            if self._list_of_pointers:
                c_type += "*"
            array = ffi.new(f"{c_type}[]", val)
            size = len(val)

        else:
            if isinstance(ua_class(), UaType):
                c_type = _get_c_type(ua_class()._ptr)
                if list_of_pointers:
                    c_type += "*"
            else:
                raise AttributeError("'ua_class' has do be None or a subclass of UaType.")

            if size is None:
                raise AttributeError("if 'val' is not a python list, 'size' has to be set.")
            elif isinstance(size, SizeT):
                size = size._val

            if isinstance(val, UaType):
                array = ffi.cast(f"{c_type}[{size}]", val._ptr)
            elif val is None:
                array = ffi.new(f"{c_type}[{size}]")
            else:
                array = ffi.cast(f"{c_type}[{size}]", val)

        super().__init__(val=array, is_pointer=True)
        self._size = size
        self._ua_type = ua_class
        self._c_type = c_type

    @property
    def _ptr(self):
        """Returns a pointer to the first array item. If list_of_pointers is True this is a **"""
        return self._value

    @property
    def ua_type(self):
        return self._ua_type

    @property
    def value(self):
        """Returns a python list of the array. If list_of_pointers is True this is a list of pointers."""
        return ffi.unpack(self._ptr, self._size)

    def __len__(self):
        return self._size

    def __setitem__(self, index: int, value):
        if 0 > index or index >= self._size:
            raise KeyError("index out of bound")
        if isinstance(value, UaType):
            self._ptr[index] = value._ptr if self._list_of_pointers else value._val

    def __getitem__(self, index):
        if isinstance(index, int):
            if 0 > index or index > self._size:
                raise KeyError("index out of bound")
            return self._ua_type(val=self.value[index], is_pointer=self._list_of_pointers)
        elif isinstance(index, slice):
            UaList(self.value[index], ua_class=self._ua_type)
        else:
            raise AttributeError("invalid indexing")

    def __str__(self, n=None):
        return str(self.value) + ("" if n is None else "\n")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec  3 09:04:55 2021

@author: clement
"""

# =============================================================================
## Library dependancies
# =============================================================================

import pandas as pd
import numpy as np
import seaborn as sns
import networkx as nx
import os
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3

from mpl_toolkits.mplot3d import proj3d
from matplotlib.patches import FancyArrowPatch
from pathlib import Path
from utilitaries import toEdgeList, dfListConnex, includeSJ
from clusteringCoefficient import unweightedCC, weightedCC

# =============================================================================
## Graphical functions
# =============================================================================

###############################################################################

class Arrow3D(FancyArrowPatch):
    def __init__(self, xs, ys, zs, *args, **kwargs):
        FancyArrowPatch.__init__(self, (0,0), (0,0), *args, **kwargs)
        self._verts3d = xs, ys, zs

    def draw(self, renderer):
        xs3d, ys3d, zs3d = self._verts3d
        xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)
        self.set_positions((xs[0],ys[0]),(xs[1],ys[1]))
        FancyArrowPatch.draw(self, renderer)

###############################################################################

def network3D(dir_in,name, step):
    """
    A retester
    """

    df_lst = dfListConnex(dir_in,name)
    k = step

    fig = plt.figure(k)
    ax = p3.Axes3D(fig)
    ax.set_axis_off()
    sc = ax.scatter(df_lst[k]['x'],df_lst[k]['z'],df_lst[k]['y'],
                    s = df_lst[k]['weight']*10,
                    c = df_lst[k]['small_junction'],
                    cmap = 'bwr_r',
                    alpha=0.8)

    for cluster in df_lst[k]['junction']:
        # check if the cluster is sufficiently heavy
        # if not df_lst[k].loc[df_lst[k]['cluster'] == cluster, 'small_cluster'].item():
            L = df_lst[k].loc[df_lst[k]['junction'] == cluster, 'neighbors_list'].to_list()[0]
            for neighbour in L:
                # check if the neighbour is sufficiently heavy
                if not df_lst[k].loc[df_lst[k]['junction'] == neighbour,'small_junction'].item():
                    df_cluster = df_lst[k].loc[df_lst[k]['junction'] == cluster].copy()
                    df_neighbour = df_lst[k].loc[df_lst[k]['junction'] == neighbour].copy()
                    # return df_cluster, df_neighbour
                    arw = Arrow3D([df_cluster['x'].item(),df_neighbour['x'].item()],
                              [df_cluster['z'].item(),df_neighbour['z'].item()],
                              [df_cluster['y'].item(),df_neighbour['y'].item()],
                              color="grey",
                              lw = 1, mutation_scale=0.5,alpha=0.95)
                    ax.add_artist(arw)
                else:
                    pass
        # else:
        #     pass
    # L = [[5,0,0,'black'],[0,5,0,'green'],[0,0,5,'orange']]
    # df = pd.DataFrame(L,columns=['x','y','z','c'])
    # sc = ax.scatter(df['x'],df['y'],df['z'],
    #                 c = df['c'],
    #                 alpha=1)
    ax.view_init(0,0)

    return sc

def graphMass(dir_in, lst, loop = True, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    loop: boolean, default True
        Indicate if you want the loop counted or not.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the mass of the junctions in the
        network.
    """

    L = []
    for elem in lst:
        data_list = dfListConnex(dir_in, elem)
        for j in range(len(data_list)):
            aff = data_list[j].iloc[0]['affinity']
            M = []
            df = includeSJ(data_list[j], SJ)
            for k in range(len(df)):
                if loop:
                    M += [[aff, df.iloc[k]['mass']]]
                else:
                    M += [[aff, df.iloc[k]['degreeMG']]]
            L += M

    df = pd.DataFrame(L,columns=['aff','mass'])
    T = list(set(df['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df =  df.append({'aff': elem,
                                 'mass': -1},
                                ignore_index=True)
    df['affinity'] = df['aff'].apply(lambda x: round(1 - (x/25),2))
    figure = sns.violinplot(x="affinity",y='mass',data=df)

    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    plt.xlabel('Affinity', size=28, family='helvetica')
    if loop:
        plt.ylabel('Junction mass with loops included', size=28, family='helvetica')
    else:
        plt.ylabel('Junction mass without loops', size=28, family='helvetica')
    figure.set(ylim=(0, None))
    return figure

def graphDegree(dir_in, lst, multi = True, SJ = False):
    """
    Return a seaborn plot showing the degree or multoidegree of the junction
    with or without the loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    multi: boolean, default True
        Indicate if you want the degree or the multidegree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not. The degree
        (or multidegree) of a small junction is always 0.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the degree or multidegree of the
        junctions in the network.
    """

    L = []
    for elem in lst:
        data_list = dfListConnex(dir_in, elem)
        for j in range(len(data_list)):
            aff = data_list[j].iloc[0]['affinity']
            M = []
            df = includeSJ(data_list[j], SJ)
            for k in range(len(df)):
                if multi:
                    M += [[aff, df.iloc[k]['degreeMG']]]
                else:
                    M += [[aff, df.iloc[k]['degreeG']]]
            L += M

    df = pd.DataFrame(L,columns=['aff','degree'])
    T = list(set(df['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df =  df.append({'aff': elem,
                                 'degree': -1},
                                ignore_index=True)
    df['affinity'] = df['aff'].apply(lambda x: round(1 - (x/25),2))

    figure = sns.violinplot(x="affinity",y='degree',data=df)

    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    plt.xlabel('Affinity', size=28, family='helvetica')
    if multi:
        plt.ylabel('Multidegree', size=28, family='helvetica')
    else:
        plt.ylabel('Degree', size=28, family='helvetica')
    figure.set(ylim=(0, None))
    return figure

def graphClustering(dir_in, lst, unweighted = True, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    unweighted: boolean, default True
        Indicate if you want the weighted or unweighted degree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the average clustering coefficient
        of the network.
    """

    df_clustering = pd.DataFrame()
    df_random = pd.DataFrame()

    affinityList = []
    typeList = []

    affinityRandomList = []
    CCList = []
    CCRandomList = []
    runList = []
    runRandomList = []
    if unweighted:
        for name in lst:
            df_lst = dfListConnex(dir_in, name)
            n = len(df_lst)
            try:
                affinityList += [df_lst[0].at[0,'affinity']]*n
            except KeyError:
                print(name)
                break
            typeList += [str(df_lst[0].at[0,'type'])]*n
            affinityRandomList += ['small world']*n
            #label = df_lst[0].at[0,'label']
            for k in range(len(df_lst)):
                df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
                G, avgUWCC = unweightedCC(df_edges)

                # Get the parameters
                n = len(df_lst[k])
                L = list(G.degree())
                M = [x[1] for x in L]
                m = sum(M)/2

                # Create a small-world graph and exploit
                k = int(np.ceil((m//n) + 1))
                K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

                avgSW = nx.average_clustering(K)

                CCList += [avgUWCC]
                CCRandomList += [avgSW]
                runList += [df_lst[k].at[0,'label']]
                runRandomList += ['small world']
    else:
        for name in lst:
            df_lst = dfListConnex(dir_in, name)
            n = len(df_lst)
            try:
                affinityList += [df_lst[0].at[0,'affinity']]*n
            except KeyError:
                print(name)
                break
            typeList += [str(df_lst[0].at[0,'type'])]*n
            # affinityRandomList += ['small world']*n
            for k in range(len(df_lst)):
                df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
                G, avgWCC = weightedCC(df_edges)

                # # Get the parameters
                # n = len(df_lst[k])
                # L = list(G.degree())
                # M = [x[1] for x in L]
                # m = sum(M)/2

                # # Create a small-world graph and exploit
                # k = int(np.ceil((m//n) + 1))
                # K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

                # degSW = nx.degree(K)
                # avgSW = 0
                # for key in K.nodes():
                #     avgSW += degSW[key]
                # avgSW /= len(degSW)

                CCList += [avgWCC]
                # CCRandomList += [avgSW]
                runList += [name]
                # runRandomList += ['small world']

    df_random['type'] = affinityRandomList
    df_random['affinity'] = affinityRandomList
    df_random['run'] = runRandomList
    df_random['CC'] = CCRandomList

    # newRunList = []
    # for elem in runList:
    #     if elem == '6E48/14001':
    #         newRunList += ['A']
    #     elif elem == '6E48/14002':
    #         newRunList += ['B']
    #     elif elem == '6E48/14003':
    #         newRunList += ['C']
    #     elif elem == '6E48/14004':
    #         newRunList += ['D']
    #     elif elem == '7E48/14001':
    #         newRunList += ['A two off']
    #     elif elem == '7E48/14002':
    #         newRunList += ['B two off']
    #     elif elem == '7E48/14003':
    #         newRunList += ['C two off']
    #     elif elem == '7E48/14004':
    #         newRunList += ['D two off']

    df_clustering['type'] = typeList
    df_clustering['aff'] = affinityList
    df_clustering['run'] = runList
    df_clustering['CC'] = CCList

    T = list(set(df_clustering['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clustering =  df_clustering.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clustering['affinity'] = df_clustering['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    #df_clustering = df_clustering.sort_values(by=['run'])

    df_random = df_random.append(df_clustering,ignore_index=True)
    sns.set_style({'axes.grid': True})
    figure = sns.violinplot(x = 'run', y = 'CC',
                     scale = "width",
                     data = df_clustering
                     )

    # Set the graphical properties
    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # Labels for the polymer type and concentration graphs
    # ax.set_xticks(range(len(df_random['run'].unique()))) # <--- set the ticks first
    # ax.set_xticklabels(['small world',
    #                      '2B48 - 0.0008', '0.001', '0.002', '0.004', '0.006', '0.008',
    #                      '4B16 - 0.0005', '0.001', '0.002', '0.004',
    #                      '5B8 - 0.0004', '0.0006', '0.0008', '0.001',
    #                      '6B6 - 0.0004', '0.0006', '0.0008', '0.001'
    #                      ])
    plt.xticks(rotation=19, horizontalalignment='right')

    plt.xlabel('Polymer Type and concentration', size=28, family='helvetica')

    if unweighted:
        plt.ylabel('Average clustering coefficient', size=28, family='helvetica')
        figure.set(ylim=(0,1))
    else:
        plt.ylabel('Average weighted clustering coefficient', size=28, family='helvetica')
        figure.set(ylim=(0,None))

def graphClusteringComparison(dir_in, lst, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    unweighted: boolean, default True
        Indicate if you want the weighted or unweighted degree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the average clustering coefficient
        of the network.
    """

    df_clusteringCC = pd.DataFrame()
    df_random = pd.DataFrame()

    CCaffinityList = []
    CCtypeList = []

    affinityRandomList = []
    CCList = []
    CCRandomList = []
    CCrunList = []
    runRandomList = []

    for name in lst:
        df_lst = dfListConnex(dir_in, name)
        n = len(df_lst)
        try:
            CCaffinityList += [df_lst[0].at[0,'affinity']]*n
        except KeyError:
            print(name)
            break
        CCtypeList += [str(df_lst[0].at[0,'type'])]*n
        affinityRandomList += ['small world']*n
        for k in range(len(df_lst)):
            df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
            G, avgUWCC = unweightedCC(df_edges)

            # Get the parameters
            n = len(df_lst[k])
            L = list(G.degree())
            M = [x[1] for x in L]
            m = sum(M)/2

            # Create a small-world graph and exploit
            k = int(np.ceil((m//n) + 1))
            K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

            avgSW = nx.average_clustering(K)

            CCList += [avgUWCC]
            CCRandomList += [avgSW]
            CCrunList += [name]
            runRandomList += ['small world']

    df_random['type'] = affinityRandomList
    df_random['affinity'] = affinityRandomList
    df_random['run'] = runRandomList
    df_random['CC'] = CCRandomList

    df_clusteringCC['type'] = CCtypeList
    df_clusteringCC['aff'] = CCaffinityList
    df_clusteringCC['run'] = CCrunList
    df_clusteringCC['CC'] = CCList

    T = list(set(df_clusteringCC['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clusteringCC =  df_clusteringCC.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clusteringCC['affinity'] = df_clusteringCC['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    df_clusteringCC = df_clusteringCC.sort_values(by=['type'])

    df_random = df_random.append(df_clusteringCC,ignore_index=True)

    f, (subfigure1, subfigure2) = plt.subplots(1, 2, gridspec_kw={'width_ratios': [5, 4]})
    sns.set_style({'axes.grid': True})
    plt.rcParams["axes.titlesize"] = 25
    sns.violinplot(x = 'type', y = 'CC',
                     scale = "width",
                     data = df_random,
                     ax = subfigure1
                     )

    # Set the graphical properties
    for tick in subfigure1.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in subfigure1.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # WCC
    df_clusteringWCC = pd.DataFrame()

    WCCaffinityList = []
    WCCtypeList = []

    affinityRandomList = []
    WCCList = []
    WCCrunList = []

    for name in lst:
        df_lst = dfListConnex(dir_in, name)
        n = len(df_lst)
        try:
            WCCaffinityList += [df_lst[0].at[0,'affinity']]*n
        except KeyError:
            print(name)
            break
        WCCtypeList += [str(df_lst[0].at[0,'type'])]*n
        for k in range(len(df_lst)):
            df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
            G, avgWCC = weightedCC(df_edges)

            WCCList += [avgWCC]
            WCCrunList += [name]

    df_clusteringWCC['type'] = WCCtypeList
    df_clusteringWCC['aff'] = WCCaffinityList
    df_clusteringWCC['run'] = WCCrunList
    df_clusteringWCC['CC'] = WCCList

    T = list(set(df_clusteringWCC['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clusteringWCC =  df_clusteringWCC.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clusteringWCC['affinity'] = df_clusteringWCC['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    df_clusteringWCC = df_clusteringWCC.sort_values(by=['type'])

    sns.set_style({'axes.grid': True})
    sns.violinplot(x = 'type', y = 'CC',
                     scale = "width",
                     data = df_clusteringWCC,
                     ax = subfigure2
                     )

    # Set the graphical properties
    for tick in subfigure2.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in subfigure2.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # Labels for the polymer type and concentration graphs
    # ax.set_xticks(range(len(df_random['run'].unique()))) # <--- set the ticks first
    # ax.set_xticklabels(['small world',
    #                      '2B48 - 0.0008', '0.001', '0.002', '0.004', '0.006', '0.008',
    #                      '4B16 - 0.0005', '0.001', '0.002', '0.004',
    #                      '5B8 - 0.0004', '0.0006', '0.0008', '0.001',
    #                      '6B6 - 0.0004', '0.0006', '0.0008', '0.001'
    #                      ])
    # plt.xticks(rotation=19, horizontalalignment='right')

    subfigure1.set_xlabel('Polymer Type', size=28, family='helvetica')
    subfigure1.set_ylabel('Average clustering coefficient', size=28, family='helvetica')
    subfigure1.set(ylim=(0,1))

    subfigure2.set_xlabel('Polymer Type', size=28, family='helvetica')
    subfigure2.set_ylabel('Average weighted clustering coefficient', size=28, family='helvetica')
    subfigure2.set(ylim=(0,None))

    return f.tight_layout()

def graphFluidity(dir_in, filename):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the file filename.
    filename: string
        Name of the file that will de converted in a dataframe and where we
        already have saved the fluidity (it's a time consuming 'function thats
        why it's already done).

    Returns:
    --------
    figure: seaborn lineplot
        The plot showing the evolution of the average fluidity of the network.
    """

    dir_path = Path(os.getcwd())
    file_path = dir_path.parent.joinpath(dir_in, filename)
    df = pd.read_pickle(file_path)

    figure = sns.lineplot(x = 'step', y = 'fluidity',
                     hue = "affinity",
                     data = df
                     )

    return figure

# =============================================================================
## List to test
# =============================================================================

P = ['1011', '2414', '2450', '622', '2214', '2806']
Q = ['2B/6091', '2B/6092', '2B/6093', '2B/6094', '2B/6095', '2B/6096',
     '4C/4025', '4C/4026', '4C/4027', '4C/4028',
     '5D/11001', '5D/11002', '5D/11003', '5D/11004',
     '6E/14001', '6E/14002', '6E/14003', '6E/14004']
R = ['2B/6091',
     '4C/4025',
     '5D/11003',
     '6E/14003']
S = ['6E48/14001','7E48/14001',
     '6E48/14002','7E48/14002',
     '6E48/14003','7E48/14003',
     '6E48/14004','7E48/14004']
T = ['4/6e48/14001', '4/6e48/14002', '4/6e48/14003', '4/6e48/14004',
     '4/6f48/15001', '4/6f48/15002', '4/6f48/15003', '4/6f48/15004',
     #'65/6d48/13001r6', '65/6d48/13002r6', '65/6d48/13003r6', '65/6d48/13004r6',
     '65/6e48/14001', '65/6e48/14002', '65/6e48/14003', '65/6e48/14004',
     '65/6d48/13001', '65/6d48/13002', '65/6d48/13003', '65/6d48/13004',
     '65/6f48/15001', '65/6f48/15002', '65/6f48/15003', '65/6f48/15004'
     ]

# =============================================================================
## Graphs generations
# =============================================================================

graphClustering('neo_junctions',T[:8])


import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn import tree
from sklearn.datasets import load_digits

dataDg = load_digits()
# print(dir(dataDg))
# print(len(dataDg['target']))

dfDg = pd.DataFrame(
    dataDg['data'],
    columns=np.arange(len(dataDg['data'][0]))
)
dfDg['target']=dataDg['target']
# print(dfDg)

letarget = LabelEncoder()
dfDg['letarget'] = letarget.fit_transform(dfDg['target'])

x = dfDg.drop(['target','letarget'],axis=1)
y = dfDg['letarget']
model = tree.DecisionTreeClassifier()
model.fit(x,y)

# print(x.iloc[0])
# print(y.iloc[0])
# tree.export_graphviz(
#     model.fit(x,y),
#     out_file='digits.dot',
#     feature_names=x.iloc[0],
#     class_names=y.iloc[0]
# )


""" An utility class where various functions are stored.
These do not fit in any other class, according to the principle of encapsulation. """
import random
import os

hexadecimal_string = '0123456789abcdef'

def generate_random_string(length):
    """ Generate a random string of given length.
    :param length: The length of the string to be generated.
    :return: A random string of given length. """

    return ''.join(random.choice(hexadecimal_string) for _ in range(length))

def delete_file(file_path):
    """ Delete a file.
    :param file_path: The path of the file to be deleted. """

    try:
        os.remove(file_path)
    except OSError:
        print("Cannot delete: no such file.")


# -*- coding: utf-8 -*-
"""
Created on Thu May 27 09:14:59 2021
@author: CanWang
1. Save all geometry related subFunctions in slipGeom.py file
2.
"""
import numpy as np
def slipSurface(Nmax,Mmax,Rnmax,R0,x0,y0,z0):
    Tmode = 2.5
    #This function was used to generate 3D rotational surface;
    Nmax1 = Nmax+1
    CircleCen = np.zeros([Nmax1,3])
    GridPos = np.zeros([Mmax*Nmax1,5])

    xpos = np.zeros([Mmax,Nmax1])
    ypos = np.zeros([Mmax,Nmax1])
    zpos = np.zeros([Mmax,Nmax1])
    Rs = np.zeros([Nmax1,1])
    theta = np.zeros([Nmax1,1])
    alpha = np.zeros([Mmax,1])
    xc = np.zeros([Nmax1,1])
    yc = np.zeros([Nmax1,1])
    zc = np.zeros([Nmax1,1])
    ppos = np.zeros([Mmax*Nmax1,3])
    xp = np.zeros([Mmax,Nmax1])
    yp = np.zeros([Mmax,Nmax1])
    zp = np.zeros([Mmax,Nmax1])
    inc = 0

    for i in range (0,Nmax1): # i step is the one at each rotation angle
        if i < Nmax:
            theta[i,] = np.pi/2*i/Nmax
            Rs[i,] = R0+4*(Rnmax-R0)/np.pi**2*theta[i,]**2
            xc[i,] = x0
            yc[i,] = y0 - np.cos(i*np.pi/2/Nmax)*(R0+(Rnmax - R0)*(i/Nmax)**Tmode)
            zc[i,] = z0 - np.sin(i*np.pi/2/Nmax)*(R0+(Rnmax - R0)*(i/Nmax)**Tmode)
        else:
            Rs[i,] = Rnmax
            theta[i,] = np.pi/2
            xc[i,] = 0
            yc[i,] = 0
            zc[i,] = 0
        CircleCen[i,0] = xc[i,]
        CircleCen[i,1] = yc[i,]
        CircleCen[i,2] = zc[i,]
        for j in range (0,Mmax):# j step is the one at each circle plane
            alpha[j,] = 2*np.pi*j/Mmax
            xp[j,i] = xc[i,]+Rs[i,]*np.sin(alpha[j,])
            yp[j,i] = yc[i,]+Rs[i,]*np.cos(alpha[j,])
            zp[j,i] = zc[i,]
            xpos[j,i] = x0 + np.sin(2*np.pi*j/Mmax)*(R0 + (Rnmax - R0)*(i/Nmax)**Tmode)
            ypos[j,i] = y0 - np.cos(np.pi*i/2/Nmax)*(1-np.cos(2*np.pi*j/Mmax))*(R0+(Rnmax-R0)*(i/Nmax)**Tmode)
            zpos[j,i] = z0 - np.sin(np.pi*i/2/Nmax)*(1-np.cos(2*np.pi*j/Mmax))*(R0+(Rnmax-R0)*(i/Nmax)**Tmode)
            ppos[inc,0] = xp[j,i]
            ppos[inc,1] = yp[j,i]
            ppos[inc,2] = zp[j,i]
            GridPos[inc,0] = xpos[j,i]
            GridPos[inc,1] = ypos[j,i]
            GridPos[inc,2] = zpos[j,i]
            GridPos[inc,3] = theta[i,]
            GridPos[inc,4] = alpha[j,]
            inc = inc + 1
    return(GridPos,xpos,ypos,zpos,CircleCen)

def wedgeNode(Nmax,Mmax,Rnmax,R0,x0,y0,z0):
    #call subFunction slipSurface
    #global meshx,meshy,meshz
    '''
    Prepare sequence of all nodes in a wedge defined by 123456.
    Rules of Nodes preparation in the mesh connection:
    1. Node 1,2,3,4 are in the primeter. node 5 and 6 are in the centre;
    2. Node 1 to 2 and node 3 to 4 are clockwise;
    3. Node 1 is above 3. node 2 is above 4;
    4. Node 5 is above 6;
    '''
    CircleCen = slipSurface(Nmax,Mmax,Rnmax,R0,x0,y0,z0)[4]
    meshx = np.zeros([Nmax*Mmax,7])
    meshy = np.zeros([Nmax*Mmax,7])
    meshz = np.zeros([Nmax*Mmax,7])
    mesh =  [meshx,meshy,meshz]
    mpos = [slipSurface(Nmax,Mmax,Rnmax,R0,x0,y0,z0)[1],\
            slipSurface(Nmax,Mmax,Rnmax,R0,x0,y0,z0)[2],\
            slipSurface(Nmax,Mmax,Rnmax,R0,x0,y0,z0)[3]]
    mcpos = [CircleCen[:,0],CircleCen[:,1],CircleCen[:,2]]
    #global meshid
    for i in range (0,Nmax):
        for j in range(0,Mmax):
            for k in range (0,3):
                inc = i*Mmax+j
                if j < Mmax-1:
                    mesh[k][inc,0] = inc
                    mesh[k][inc,1] = mpos[k][j,i]
                    mesh[k][inc,2] = mpos[k][j+1,i]
                    mesh[k][inc,3] = mpos[k][j,i+1]
                    mesh[k][inc,4] = mpos[k][j+1,i+1]
                    mesh[k][inc,5] = mcpos[k][i,]
                    mesh[k][inc,6] = mcpos[k][i+1,]

                else:
                    mesh[k][inc,0] = inc
                    mesh[k][inc,1] = mpos[k][j,i]
                    mesh[k][inc,2] = mpos[k][0,i]
                    mesh[k][inc,3] = mpos[k][j,i+1]
                    mesh[k][inc,4] = mpos[k][0,i+1]
                    mesh[k][inc,5] = mcpos[k][i,]
                    mesh[k][inc,6] = mcpos[k][i+1,]
    return(mesh,mcpos)

def det(a,b,c): #Determinte of a 3*3 matrix
    det = a[0]*(b[1]*c[2]-b[2]*c[1])-a[1]*(b[0]*c[2]-b[2]*c[0])+a[2]*(b[0]*c[1]-b[1]*c[0])
    return(det)

def volTetra(node1,node2,node3,node4,node5,node6):
    '''
    Calculate volume of a wedge;
    A wedge is divided by 3 Tetrahedrons of 1-2-3-5; 2-3-4-6; 2-3-5-6;
    Volume of each tetrahedron is calculate by determinate
    '''
    'Tetrahedron1 of 1-2-3-5'
    a1 = [node2[0]-node1[0],node2[1]-node1[1],node2[2]-node1[2]]
    b1 = [node3[0]-node1[0],node3[1]-node1[1],node3[2]-node1[2]]
    c1 = [node5[0]-node1[0],node5[1]-node1[1],node5[2]-node1[2]]
    vol1 = 1/6*abs(det(a1,b1,c1))

    'Tetrahedron2 of 2-3-4-6'
    a2 = [node3[0]-node2[0],node3[1]-node2[1],node3[2]-node2[2]]
    b2 = [node4[0]-node2[0],node4[1]-node2[1],node4[2]-node2[2]]
    c2 = [node6[0]-node2[0],node6[1]-node2[1],node6[2]-node2[2]]
    vol2 = 1/6*abs(det(a2,b2,c2))

    'Tetrahedron3 of 2-3-5-6'
    a3 = [node3[0]-node2[0],node3[1]-node2[1],node3[2]-node2[2]]
    b3 = [node5[0]-node2[0],node5[1]-node2[1],node5[2]-node2[2]]
    c3 = [node6[0]-node2[0],node6[1]-node2[1],node6[2]-node2[2]]
    vol3 = 1/6*abs(det(a3,b3,c3))

    'Total volume composed of 3 Tetras'
    volume = vol1 + vol2 + vol3
    return(vol1, vol2, vol3, volume)

def cenTetra(node1,node2,node3,node4,node5,node6):
    #find center position of 3 tetras in the wedge
    'Tetrahedron1: centre of 1-2-3-5'
    x1 = 1/4*(node1[0]+node2[0]+node3[0]+node5[0])
    y1 = 1/4*(node1[1]+node2[1]+node3[1]+node5[1])
    z1 = 1/4*(node1[2]+node2[2]+node3[2]+node5[2])

    'Tetrahedron2: centre of 2-3-4-6'
    x2 = 1/4*(node2[0]+node3[0]+node4[0]+node6[0])
    y2 = 1/4*(node2[1]+node3[1]+node4[1]+node6[1])
    z2 = 1/4*(node2[2]+node3[2]+node4[2]+node6[2])

    'Tetrahedron3: centre of 2-3-5-6'
    x3 = 1/4*(node2[0]+node3[0]+node5[0]+node6[0])
    y3 = 1/4*(node2[1]+node3[1]+node5[1]+node6[1])
    z3 = 1/4*(node2[2]+node3[2]+node5[2]+node6[2])
    return(x1,y1,z1,x2,y2,z2,x3,y3,z3)

def wedgePos(Nmax,Mmax,Rnmax,R0,x0,y0,z0):
    #find center position of the wedge
    '''
    call subFunction wedgeNode
    call subFunction volTetra
    call subFunction cenTetra
    '''
    TVol = np.zeros([Nmax*Mmax,10])
    TCen = np.zeros([Nmax*Mmax,10])
    wedge = np.zeros([Nmax*Mmax,10]) #represents centre of wedge
    mesh = wedgeNode(Nmax,Mmax,Rnmax,R0,x0,y0,z0)[0]
    meshx = mesh[0]
    meshy = mesh[1]
    meshz = mesh[2]
    for i in range(0, Nmax):
        for j in range(0, Mmax):
            k = i*Mmax+j
            '''
            Tetrahedron 1 of points 1-2-3-5
            Tetrahedron 2 of points 2-3-4-6
            Tetrahedron 3 of points 2-3-5-6

            Volume of the Tetrahedron takes symmertric shapes
            Note if  k < Mmax/2, Node order is 1-2-3-4-5-6
                 if  k >= Mmax/2, Node order is 2-1-4-3-5-6.
                     The volume becomes symmetric.
            '''
            node1 = [meshx[k,1],meshy[k,1],meshz[k,1]]
            node2 = [meshx[k,2],meshy[k,2],meshz[k,2]]
            node3 = [meshx[k,3],meshy[k,3],meshz[k,3]]
            node4 = [meshx[k,4],meshy[k,4],meshz[k,4]]
            node5 = [meshx[k,5],meshy[k,5],meshz[k,5]]
            node6 = [meshx[k,6],meshy[k,6],meshz[k,6]]

            TVol[k,0] = k

            for m in range(0,4):
                if j < Mmax/2:
                    TVol[k,m+1] = volTetra(node1,node2,node3,node4,node5,node6)[m]
                else:
                    TVol[k,m+1] = volTetra(node2,node1,node4,node3,node5,node6)[m]

            TCen[k,0] = k

            for m in range(0,9):
                if j < Mmax/2:
                    TCen[k,m+1] = cenTetra(node1,node2,node3,node4,node5,node6)[m]
                else:
                    TCen[k,m+1] = cenTetra(node2,node1,node4,node3,node5,node6)[m]

            wedge[k,0] = k
            wedge[k,1] = (TVol[k,1]*TCen[k,1] + TVol[k,2]*TCen[k,4] + \
                         TVol[k,3]*TCen[k,7])/TVol[k,4]
            wedge[k,2] = (TVol[k,1]*TCen[k,2] + TVol[k,2]*TCen[k,5] + \
                         TVol[k,3]*TCen[k,8])/TVol[k,4]
            wedge[k,3] = (TVol[k,1]*TCen[k,3] + TVol[k,2]*TCen[k,6] + \
                         TVol[k,3]*TCen[k,9])/TVol[k,4]
            wedge[k,4] = TVol[k,4]

    return(wedge)

def melonPos(Nmax,Mmax,wedge):
    #find centre position of the melon
    melon = np.zeros([Nmax,5])
    for i in range (0,Nmax):
        xc = 0
        yc = 0
        zc = 0
        vol = 0
        for j in range (0,Mmax):
            vol = vol + wedge[i*Mmax+j,4]

        for j in range (0,Mmax):
            xc = xc + wedge[i*Mmax+j,1]*wedge[i*Mmax+j,4]/vol
            yc = yc + wedge[i*Mmax+j,2]*wedge[i*Mmax+j,4]/vol
            zc = zc + wedge[i*Mmax+j,3]*wedge[i*Mmax+j,4]/vol

        melon[i,0] = i
        melon[i,1] = xc
        melon[i,2] = yc
        melon[i,3] = zc
        melon[i,4] = vol
    return(melon)

"""
SubFunction tri1Surf and tri2Surf:
find tricentre and trinorm of surface triangles;
triangle1 = node2-1-3, clockwise
triangle2 = node2-3-4, clockwise
tricentre = [meshid, triangle1 area, triangle1 centre, triangle2 area, triangle2 centre]
trinorm = [meshid,triangle norm1, triangle norm2]
"""

def tri1Surf(head,i,Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh):
    """
    For the head and tail side by one surface of three nodes
    cross = | i, j, k|
            |a1,a2,a3|
            |b1,b2,b3|
          = |a2*b3 - a3*b2|*i-|a1*b3-a3*b1|*j+|a1*b2-a2*b1|*k
    dot(|a1,a2,a3|,|b1,b2,b3|) = a1*b1+a2*b2+a3*b3
    vector01 = np.array([n1[0]-n0[0],n1[1]-n0[1],n1[2]-n0[2]])
    vector12 = np.array([n2[0]-n1[0],n2[1]-n1[1],n2[2]-n1[2]])
    dot.product = v01[0]*v12[0]+v01[1]*v12[1]+v01[2]*v12[2]
    """

    meshx = mesh[0]
    meshy = mesh[1]
    meshz = mesh[2]
    if head == 1:
        n1 = np.array([meshx[i,1],meshy[i,1],meshz[i,1]])
        n2 = np.array([meshx[i,2],meshy[i,2],meshz[i,2]])
        n4 = np.array([meshx[i,4],meshy[i,4],meshz[i,4]])
        v21 = n1 - n2 # represents vector21
        v14 = n4 - n1
        dot = np.dot(v21,v14) #Triangle 1,2,4
        d21 = (v21[0]**2+v21[1]**2+v21[2]**2)**0.5
        d14 = (v14[0]**2+v14[1]**2+v14[2]**2)**0.5
        beta = np.arccos(dot/(d21*d14))
        area = 0.5*d21*d14*np.sin(beta)
        cross = np.cross(v21,v14)
        norm = cross/(cross[0]**2+cross[1]**2+cross[2]**2)**0.5
        centre = 1/3*(n1+n2+n4)
    else:
        n1 = np.array([meshx[i,1],meshy[i,1],meshz[i,1]])
        n2 = np.array([meshx[i,2],meshy[i,2],meshz[i,2]])
        n3 = np.array([meshx[i,3],meshy[i,3],meshz[i,3]])
        v21 = n1 - n2
        v13 = n3 - n1
        dot = np.dot(v21,v13)
        d21 = (v21[0]**2+v21[1]**2+v21[2]**2)**0.5
        d13 = (v13[0]**2+v13[1]**2+v13[2]**2)**0.5
        beta = np.arccos(dot/(d21*d13))
        area = 0.5*d21*d13*np.sin(beta)
        cross =  np.cross(v21,v13)
        norm = cross/(cross[0]**2+cross[1]**2+cross[2]**2)**0.5
        centre = 1/3*(n1+n2+n3)
    return(area,centre,norm)

def tri2Surf(i,Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh):
    """
    For a general side formed by two surfaces of four nodes
    cross = | i, j, k|
            |a1,a2,a3|
            |b1,b2,b3|
          = |a2*b3 - a3*b2|*i-|a1*b3-a3*b1|*j+|a1*b2-a2*b1|*k
    dot(|a1,a2,a3|,|b1,b2,b3|) = a1*b1+a2*b2+a3*b3
    vector01 = np.array([n1[0]-n0[0],n1[1]-n0[1],n1[2]-n0[2]])
    vector12 = np.array([n2[0]-n1[0],n2[1]-n1[1],n2[2]-n1[2]])
    dot.product = v01[0]*v12[0]+v01[1]*v12[1]+v01[2]*v12[2]
    """
    meshx,meshy,meshz = [mesh[0],mesh[1],mesh[2]]
    n1 = np.array([meshx[i,1],meshy[i,1],meshz[i,1]])
    n2 = np.array([meshx[i,2],meshy[i,2],meshz[i,2]])
    n3 = np.array([meshx[i,3],meshy[i,3],meshz[i,3]])
    n4 = np.array([meshx[i,4],meshy[i,4],meshz[i,4]])
    v21 = n1 - n2 # represents vector21
    v13 = n3 - n1
    v23 = n3 - n2
    v34 = n4 - n3
    dot1 = np.dot(v21,v13) # Triangle 2,1,3
    dot2 = np.dot(v23,v34) # Triangle 2,3,4
    d21 = (v21[0]**2+v21[1]**2+v21[2]**2)**0.5
    d13 = (v13[0]**2+v13[1]**2+v13[2]**2)**0.5
    d23 = (v23[0]**2+v23[1]**2+v23[2]**2)**0.5
    d34 = (v34[0]**2+v34[1]**2+v34[2]**2)**0.5
    beta1 = np.arccos(dot1/(d21*d13))
    beta2 = np.arccos(dot2/(d23*d34))

    area1 = 0.5*d21*d13*np.sin(beta1)
    area2 = 0.5*d23*d34*np.sin(beta2)
    cross1 = np.cross(v21,v13)
    cross2 = np.cross(v23,v34)
    norm1 = cross1/(cross1[0]**2+cross1[1]**2+cross1[2]**2)**0.5
    norm2 = cross2/(cross2[0]**2+cross2[1]**2+cross2[2]**2)**0.5
    centre1 = 1/3*(n2+n1+n3)
    centre2 = 1/3*(n2+n3+n4)
    return(area1,area2,centre1,centre2,norm1,norm2)

def surfNorm(Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh):
    '''
    To find the norm of each triangular facet
    If a triangle is at the head of the circle, degenerate to triangle 1,2,4
    If a triangle is at the tail of the circle, degenerate to triangle 1,2,3
    '''
    # tricentre = np.zeros([Nmax*Mmax,9])
    nVec = np.zeros([Nmax*Mmax,7])
    for i in range (0,Nmax*Mmax):
        if i%Mmax == 0:
            head = 1
            matrix = tri1Surf(head,i,Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh)
            nVec[i,0] = i
            nVec[i,4] = matrix[2][0]
            nVec[i,5] = matrix[2][1]
            nVec[i,6] = matrix[2][2]

        elif (i+1)%Mmax == 0:
            head = 0
            matrix = tri1Surf(head,i,Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh)
            nVec[i,0] = i
            nVec[i,1] = matrix[2][0]
            nVec[i,2] = matrix[2][1]
            nVec[i,3] = matrix[2][2]

        else:
            matrix = tri2Surf(i,Nmax,Mmax,Rnmax,R0,x0,y0,z0,mesh)
            nVec[i,0] = i
            nVec[i,1] = matrix[4][0]
            nVec[i,2] = matrix[4][1]
            nVec[i,3] = matrix[4][2]
            nVec[i,4] = matrix[5][0]
            nVec[i,5] = matrix[5][1]
            nVec[i,6] = matrix[5][2]
    return(nVec)

def n3pts(pt1,pt2,pt3):
    #anticlock wise direction
    v12 = pt2 - pt1
    v23 = pt2 - pt3
    cross = np.cross(v12,v23)
    norm = cross/(cross[0]**2+cross[1]**2+cross[2]**2)**0.5
    return (norm)

def intStPL(pt1,pt2,cp,vec):
    #Intersect point between a plane and a line
    a,b,c = [pt1[0],pt1[1],pt1[2]] #point1
    d,e,f = [pt2[0],pt2[1],pt2[2]] #point2
    x1,y1,z1 = [cp[0],cp[1],cp[2]] #centre of plane
    n1,n2,n3 = [vec[0],vec[1],vec[2]] #norm of the plane
    t = ((n1*x1+n2*y1+n3*z1)-n1*a-n2*b-n3*c)/(n1*(d-a)+n2*(e-b)+n3*(f-c))
    x,y,z = [t*(d-a)+a, t*(e-b)+b, t*(f-c)+c]
    return(x,y,z)

def ptInTri(A,B,C,P):
    #Tell if a point is inside the triangle area on the same plane
    flag = 0
    v0 = C - A
    v1 = B - A
    v2 = P - A
    dot00 = np.dot(v0,v0)
    dot01 = np.dot(v0,v1)
    dot02 = np.dot(v0,v2)
    dot11 = np.dot(v1,v1)
    dot12 = np.dot(v1,v2)

    invDenom = 1/(dot00*dot11-dot01*dot01)
    u = (dot11*dot01 - dot01*dot12)*invDenom
    v = (dot00*dot12 - dot01*dot02)*invDenom
    if u>=0 and v>=0 and u+v<1:
       flag = 1
    return(flag)

def triNorm(node1,node2,node3):
    #Looking at -z axis assume clockwise
    #Assume clockwise direction to have vertical component downwards
    n1 = node1
    n2 = node2
    n3 = node3
    v21 = n1 - n2 # represents vector21
    v13 = n3 - n1
    dot = np.dot(v13,v21) # Triangle 2,1,3
    d21 = (v21[0]**2+v21[1]**2+v21[2]**2)**0.5
    d13 = (v13[0]**2+v13[1]**2+v13[2]**2)**0.5
    beta = np.arccos(dot/(d21*d13))
    area = 0.5*d21*d13*np.sin(beta)
    cross = np.cross(v13,v21)
    norm = cross/(cross[0]**2+cross[1]**2+cross[2]**2)**0.5
    return(area,norm)

def pjtPtPos(Mmax,Nmax,wedge,CircleCen,melon,mesh,snvec):
    '''
    #Find the Gravity plane projected on the slip surface
    #wedge ID = i*Mmax+j
    Node1 is Gravity Plane intersection
    Node2 is wedge Gravity Centre
    cp is either node2 as shared by both 1-2-3 and 2-4-3
    vec is either triangle1-2-3 or 2-4-3
    Check the temporary projection scheme
    A Typical Plane for quick check is i = 6 and j = 17
    '''
    Gnvec = np.zeros([Nmax,3])
    GcInt = np.zeros([Nmax,3])
    ptPjt = np.zeros([Nmax*Mmax,10])
    meshx,meshy,meshz = [mesh[0],mesh[1],mesh[2]]
    # wedge matrix contains gravity centre and Area
    # if a = 1, pt4 in triangle 123, else in triangle 234
    # Project [0] - [2] Gravity + GeomCentre project on slip surface
    # Prorject[4] - [6] Normal of the array GeomCentre to GravityCentre
    for i in range(0,Nmax):
        m = i*Mmax + round(Mmax*0.3)
        nd1 = np.array([wedge[m,1],wedge[m,2],wedge[m,3]])
        m = i*Mmax + round(Mmax*0.6)
        nd2 = np.array([wedge[m,1],wedge[m,2],wedge[m,3]])
        m = i*Mmax + round(Mmax*0.9)
        nd3 = np.array([wedge[m,1],wedge[m,2],wedge[m,3]])
        '''
        Find Circular Centre Intersect with the Plane
        Melon is the Melon Centre Matrix
        vec is the triangle of wedge
        '''
        Gnvec[i,:] = triNorm(nd1,nd2,nd3)[1]
        nd4 = np.array([CircleCen[i,0],CircleCen[i,1],CircleCen[i,2]])
        nd5 = np.array([CircleCen[i+1,0],CircleCen[i+1,1],CircleCen[i+1,2]])
        cp1 = np.array([melon[i,1],melon[i,2],melon[i,3]])
        vec1 = np.array([Gnvec[i,0],Gnvec[i,1],Gnvec[i,2]])
        GcInt[i,0] = intStPL(nd4,nd5,cp1,vec1)[0]
        GcInt[i,1] = intStPL(nd4,nd5,cp1,vec1)[1]
        GcInt[i,2] = intStPL(nd4,nd5,cp1,vec1)[2]
        for j in range(0,Mmax):
            node1 = np.array([GcInt[i,0],GcInt[i,1],GcInt[i,2]])
            #node2 = np.array([melon[i,1],melon[i,2],melon[i,3]])
            k = i*Mmax+j
            node2 = np.array([wedge[k,1],wedge[k,2],wedge[k,3]])
            cp = np.array([meshx[k,2],meshy[k,2],meshz[k,2]])

            if k % Mmax == 0: #if the point is at the head of the circle
                #point 3 degenerate
                vec = np.array([snvec[k,4],snvec[k,5],snvec[k,6]])
                ptPjt[k,0] = intStPL(node1,node2,cp,vec)[0]
                ptPjt[k,1] = intStPL(node1,node2,cp,vec)[1]
                ptPjt[k,2] = intStPL(node1,node2,cp,vec)[2]
            elif (k+1) % Mmax == 0: #if this point is at the tail of the circle
                vec = np.array([snvec[k,1],snvec[k,2],snvec[k,3]])
                ptPjt[k,0] = intStPL(node1,node2,cp,vec)[0]
                ptPjt[k,1] = intStPL(node1,node2,cp,vec)[1]
                ptPjt[k,2] = intStPL(node1,node2,cp,vec)[2]
            else:
                vec = np.array([snvec[k,1],snvec[k,2],snvec[k,3]])
                ptPjt[k,0] = intStPL(node1,node2,cp,vec)[0]
                ptPjt[k,1] = intStPL(node1,node2,cp,vec)[1]
                ptPjt[k,2] = intStPL(node1,node2,cp,vec)[2]
                #if ptPjt is inside triangle 1-2-3
                pt1 = np.array([meshx[k,1],meshy[k,1],meshz[k,1]])
                pt2 = np.array([meshx[k,2],meshy[k,2],meshz[k,2]])
                pt3 = np.array([meshx[k,3],meshy[k,3],meshz[k,3]])
                pt4 = np.array([ptPjt[k,0],ptPjt[k,1],ptPjt[k,2]])
                flag = ptInTri(pt1,pt2,pt3,pt4)
                ptPjt[k,3] = flag

                if flag  == 1:
                    pass
                else:
                    vec = np.array([snvec[k,4],snvec[k,5],snvec[k,6]])
                    ptPjt[k,0] = intStPL(node1,node2,cp,vec)[0] #Project intersection at slip surface
                    ptPjt[k,1] = intStPL(node1,node2,cp,vec)[1]
                    ptPjt[k,2] = intStPL(node1,node2,cp,vec)[2]

            dist = ((node1[0]-node2[0])**2+(node1[1]-node2[1])**2+(node1[2]-node2[2])**2)**0.5
            ptPjt[k,4] = (node2[0]-node1[0])/dist #vector direction from slip centre to slip surface
            ptPjt[k,5] = (node2[1]-node1[1])/dist
            ptPjt[k,6] = (node2[2]-node1[2])/dist

            if node2[2] > node1[2]: # if the gravity wedge centre is higher (z2>z1) than melon geometry centre
                ptPjt[k,7] =  -ptPjt[k,4] #vector direction from higher position to lower position
                ptPjt[k,8] =  -ptPjt[k,5]
                ptPjt[k,9] =  -ptPjt[k,6]
            else:
                ptPjt[k,7] =  ptPjt[k,4]
                ptPjt[k,8] =  ptPjt[k,5]
                ptPjt[k,9] =  ptPjt[k,6]
    return(ptPjt)

def pjtPlPos(Mmax,Nmax,melon,ptPjt,mesh):
#determine gravity plane perpendicular to slip surface
    k = 0
    m = 0
    Ghpl = np.zeros([Nmax*Mmax,10])
    #GVplane = np.zeros([Nmax*Mmax,10])
    meshx,meshy,meshz = [mesh[0],mesh[1],mesh[2]]
#Store nodal cooordinates at Ghpl matrix - gravity plane perpendicular to gravity stress
    for i in range(0,Nmax*Mmax):# i is the No. of the Tetrahedron
        k = k + 1
        if k > Mmax:
            k = 0
            m = m + 1

        Ghpl[i,0] = i
        if i%Mmax == 0 or (i+1)%Mmax == 0:
            pass
        else:
            node1 = np.array([melon[m,1],melon[m,2],melon[m,3]])
            node2 = np.array([ptPjt[i-1,0],ptPjt[i-1,1],ptPjt[i-1,2]])
            node3 = np.array([ptPjt[i,0],ptPjt[i,1],ptPjt[i,2]])
            node4 = np.array([ptPjt[i+1,0],ptPjt[i+1,1],ptPjt[i+1,2]])
            norm1 = triNorm(node1,node2,node3)[1]
            norm2 = triNorm(node1,node3,node4)[1]
            node5 = np.array([meshx[i,1],meshy[i,1],meshz[i,1]])
            node6 = np.array([meshx[i,3],meshy[i,3],meshz[i,3]])
            node7 = np.array([meshx[i,2],meshy[i,2],meshz[i,2]])
            node8 = np.array([meshx[i,4],meshy[i,4],meshz[i,4]])

            Ghpl[i,1] = intStPL(node5,node6,node3,norm1)[0]
            Ghpl[i,2] = intStPL(node5,node6,node3,norm1)[1]
            Ghpl[i,3] = intStPL(node5,node6,node3,norm1)[2]

            Ghpl[i,4] = intStPL(node7,node8,node3,norm2)[0]
            Ghpl[i,5] = intStPL(node7,node8,node3,norm2)[1]
            Ghpl[i,6] = intStPL(node7,node8,node3,norm2)[2]

            dist = ((Ghpl[i,1]-Ghpl[i,4])**2 + (Ghpl[i,2]-Ghpl[i,5])**2 + (Ghpl[i,3]-Ghpl[i,6])**2)**0.5

            Ghpl[i,7] = (Ghpl[i,4] - Ghpl[i,1])/dist
            Ghpl[i,8] = (Ghpl[i,5] - Ghpl[i,2])/dist
            Ghpl[i,9] = (Ghpl[i,6] - Ghpl[i,3])/dist
    return(Ghpl)


# python imports
import sys
import re

def IsEmpty(p_string):
    # at least one non-space character
    if not re.search("\S+", str(p_string)):
        return True
    return False;

def ModelObjToPyDict(p_model_object):
    py_dict = dict()
    model_fields = p_model_object.get_iterable_fields()

    for field in model_fields:
        py_dict[field] = getattr(p_model_object, field)
    return py_dict

def IsNumber(p_value):
    if re.search("\d+\.?\d*", str(p_value)):
        return True
    return False;

def IsInteger(p_value):
    if not IsNumber(p_value):
        # would probably throw a type mismatch error at the caller
        # but that would still be better than letting this pass silently
        return None
    if re.search("\.+", str(p_value)):
        return False
    return True

"""
p_list : the list to be operated on
p_packet_size : number of records per packet
"""
def ReturnQueryPackets(p_list, p_packet_size):
    return_dict = {}
    # no data, return an empty list
    if len(p_list) < 1:
        return return_dict
    # no slice number given, return the original list
    if (int(p_packet_size) <= 1):
        return return_dict

    # number of items / numrecords per page
    num_pages = len(p_list) / int(p_packet_size)
    # if not int, then modulo > 0, then add 1 page for the remainder
    if not IsInteger(num_pages):
        num_pages = int(len(p_list) / int(p_packet_size)) + 1
    offset_1 = 0
    offset_2 = int(p_packet_size)
    for i in range(num_pages):
        curr_list = p_list[offset_1:offset_2]
        if len(curr_list) < 1:
            pass
        return_dict.update({str(i):curr_list})

        # get from the next nth iteration
        offset_1 += int(p_packet_size)
        # and the slice offset has to be double the starting index
        offset_2 = offset_1 * 2
    return return_dict


#following this tutorial: https://pythonprogramming.net/pygame-drawing-shapes-objects/?completed=/adding-score-pygame-video-game/

#imports
import pygame

#initialize pygame
pygame.init()

#make rgb colors
white = (255,255,255)
black = (0,0,0)

red = (255,0,0)
green = (0,255,0)
blue = (0,0,255)

#set up display
gameDisplay = pygame.display.set_mode((800,600))
gameDisplay.fill(black)

#change a single pixel array (located at 10,20) to the color green
pixAr = pygame.PixelArray(gameDisplay)
pixAr[10][20] = green

#draw a blue line on game display starting at (100,200) and ending at (300,450) with a width of 5
pygame.draw.line(gameDisplay, blue, (100,200), (300,450), 5)

#draw a red filled in rectangle with the corner coordinates:
#400 (top right x)
#400 (top right y)
#50 (width)
#25 (height)
pygame.draw.rect(gameDisplay,red,(400,400,50,25))

#draw white circle with top right coordinates (150,150) and center point at 75
pygame.draw.circle(gameDisplay,white,(150,150),75)

#draw green polygon with specified points
pygame.draw.polygon(gameDisplay,green, ((25,75),(76,125),(250,375),(400,25),(60,540)))

#main loop
while True:
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			pygame.quit()
			quit()
	pygame.display.update()


from pyspark import SparkConf
from pyspark.sql import SparkSession, functions

from clustering import get_clusters_for_partition

base_data_dir = "/Users/timor/projects/data/amazonlab/nasa_fire_info/BOLIVIA_DL_FIRE_M6_5242/"
csvInPath = base_data_dir + "fire_archive_M6_5242.csv"
parquetOutPath = base_data_dir + "fire_archive_M6_5242_clustered_eps005_s4.parquet"

spark_conf = SparkConf()
spark_conf.set('spark.executor.memory', '4g')
spark_conf.setAppName("furry-lab")
spark_conf.setMaster("local[*]")
spark = SparkSession.builder.config(conf=spark_conf).getOrCreate()

csv_options = {"delimiter": ",", "header": "true", "inferSchema": "true"}
# add grouping week with year
ds = spark.read.format("csv").options(**csv_options).load(csvInPath) \
  .withColumn("week", functions.concat(functions.year("acq_date"), functions.lit("_"), functions.weekofyear("acq_date")))

week_ds = ds.groupBy("week").agg(
    functions.collect_list(functions.struct(
        "latitude",
        "longitude",
        "brightness",
        "acq_date",
        "confidence",
        "bright_t31",
        "week"
    )).alias("events")
).sort("week")

clustered_ds = week_ds.rdd.mapPartitions(get_clusters_for_partition).toDF().cache()
clustered_ds.count()
clustered_ds.coalesce(1).write.parquet(parquetOutPath)


# Ke Yan, Imaging Biomarkers and Computer-Aided Diagnosis Laboratory,
# National Institutes of Health Clinical Center, July 2019
"""The truncated Densenet-121 with FPN and 3DCE"""
from collections import namedtuple

import torch
import torch.nn.functional as F
from torch import nn
import torchvision.models as models
from torchvision.models.densenet import _DenseBlock, _Transition, model_urls
import math
import torch.utils.model_zoo as model_zoo
import re
from collections import OrderedDict

from maskrcnn.config import cfg

class DenseNetCustomTrunc(nn.Module):
    """The truncated Densenet-121 with FPN and 3DCE"""
    # truncated since transition layer 3 since we find it works better in DeepLesion
    # We only keep the finest-level feature map after FPN
    def __init__(self):
        super(DenseNetCustomTrunc, self).__init__()
        name = cfg.MODEL.BACKBONE.CONV_BODY
        self.depth = int(name.split('-')[1])
        self.feature_upsample = cfg.MODEL.BACKBONE.FEATURE_UPSAMPLE

        assert self.depth in [121]
        if self.depth == 121:
            num_init_features = 64
            growth_rate = 32
            block_config = (6, 12, 24)
            self.in_dim = [64, 256, 512, 1024]
        bn_size = 4
        drop_rate = 0

        # First convolution
        self.conv0 = nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)
        self.norm0 = nn.BatchNorm2d(num_init_features)
        self.relu0 = nn.ReLU(inplace=True)
        self.pool0 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        # Each denseblock
        num_features = num_init_features
        for i, num_layers in enumerate(block_config):
            block = _DenseBlock(num_layers=num_layers, num_input_features=num_features,
                                bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate)
            self.add_module('denseblock%d' % (i + 1), block)
            num_features = num_features + num_layers * growth_rate
            if i != len(block_config) - 1:
                trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2)
                self.add_module('transition%d' % (i + 1), trans)
                num_features = num_features // 2
        cfg.runtime_info.backbone_ft_dim = self.in_dim[-1]

        # Final batch norm
        # self.add_module('norm5', nn.BatchNorm2d(num_features))

        # Official init from torch repo.
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight.data)
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

        if self.feature_upsample:
            self.out_dim = cfg.MODEL.BACKBONE.OUT_CHANNELS
            self.fpn_finest_layer = cfg.MODEL.BACKBONE.FEATURE_UPSAMPLE_LEVEL-1
            for p in range(4, self.fpn_finest_layer - 1, -1):
                layer = nn.Conv2d(self.in_dim[p - 1], self.out_dim, 1)
                name = 'lateral%d' % p
                self.add_module(name, layer)

                nn.init.kaiming_uniform_(layer.weight, a=1)
                nn.init.constant_(layer.bias, 0)
            cfg.runtime_info.backbone_ft_dim = self.out_dim

        self.indim_ilf = [64, 128, 256]
        self.num_image = cfg.INPUT.NUM_IMAGES_3DCE
        self.feature_fusion_level_list = cfg.MODEL.BACKBONE.FEATURE_FUSION_LEVELS
        for p in range(len(self.feature_fusion_level_list)):
            if self.feature_fusion_level_list[p]:
                layer = nn.Conv2d(self.num_image * self.indim_ilf[p], self.indim_ilf[p], 1)
                self.add_module('conv_ilf%d'%p, layer)

    def inter_layer_fuse(self, ft, level):
        """Improved 3DCE, 3D feature fusion"""
        if self.feature_fusion_level_list[level]:
            ft_ilf = ft.reshape(-1, self.num_image * self.indim_ilf[level], ft.shape[2], ft.shape[3])
            ft_ilf = getattr(self, 'conv_ilf%d'%level)(ft_ilf)
            ft_out = ft.clone()
            ft_out[int(self.num_image/2)::self.num_image] = ft_ilf
            return ft_out
        else:
            return ft

    def forward(self, x):
        x = self.conv0(x)
        x = self.norm0(x)
        relu0 = self.relu0(x)
        pool0 = self.pool0(relu0)
        pool0_ilf = self.inter_layer_fuse(pool0, 0)

        db1 = self.denseblock1(pool0_ilf)
        ts1 = self.transition1(db1)
        ts1_ilf = self.inter_layer_fuse(ts1, 1)

        db2 = self.denseblock2(ts1_ilf)
        ts2 = self.transition2(db2)
        ts2_ilf = self.inter_layer_fuse(ts2, 2)

        db3 = self.denseblock3(ts2_ilf)

        # truncated since here since we find it works better in DeepLesion
        # ts3 = self.transition3(db3)
        # db4 = self.denseblock4(ts3)

        if self.feature_upsample:
            ftmaps = [relu0, db1, db2, db3]
            x = self.lateral4(ftmaps[-1])
            for p in range(3, self.fpn_finest_layer - 1, -1):
                x = F.interpolate(x, scale_factor=2, mode="nearest")
                y = ftmaps[p-1]
                lateral = getattr(self, 'lateral%d' % p)(y)
                x += lateral
            return [x]
        else:
            return [db3]

    def load_pretrained_weights(self):
        pattern = re.compile(
            r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$')
        state_dict = model_zoo.load_url(model_urls['densenet121'])
        for key in list(state_dict.keys()):
            res = pattern.match(key)
            if res:
                new_key = res.group(1) + res.group(2)
                state_dict[new_key] = state_dict[key]
                del state_dict[key]
        state_dict1 = {}
        for key in list(state_dict.keys()):
            new_key = key.replace('features.', '')
            state_dict1[new_key] = state_dict[key]

        self.load_state_dict(state_dict1, strict=False)

    def freeze(self):
        for name, param in self.named_parameters():
            print('freezing', name)
            param.requires_grad = False


import sys
import re

name = sys.argv[1]
fileIn = open(name)
fileOut = name.replace("retrieve","formatted")
lines = [i.rstrip('\n') for i in fileIn]

def processStr(line):
	s = ""
	linkTag = 0
	userTag = 0
	codeTag = 0
	emojiTag = 0
	dotTag = 0			#for stuff like "i need help..... help" get rid of extra dots
	codeCount = 0
	lineLen = len(line)
	for i,c in enumerate(line):
		# get rid of links
		if line[i] == "<" and i < lineLen-2:
			if line[i+1] == "h":
				linkTag = 1
			elif line[i+1] == "@":
				userTag = 1
		elif line[i] == ">":
			if linkTag and userTag:
				print("WRONG IN USER AND LINK TAG!!!!")
			elif linkTag:
				linkTag = 0
			elif userTag:
				s+=","
				userTag = 0

		# # get rid of emojis
		elif line[i] == ':' and not linkTag and i < lineLen-2:
			if not emojiTag and line[i+1]!=" " :
				emojiTag = 1
			else:
				s+="."
				emojiTag = 0
		elif not linkTag and not userTag and not emojiTag:
			s+=line[i]

	return s

with open(fileOut,'w') as o:
	for line in lines:
		s = processStr(line)
		if len(s) != 0:
			o.	write(s)
			# print(s, end= "")


from pathlib import Path
from random import sample
from typing import List, Union
import shelve
import time
import wx
import wx.adv
import constants as C
import options_gui
from common import OptionsData, ResultsData
from data_gui import ResultsFrame

class GenLotteryMainFrame(options_gui.MainFrame):
    """ A Controller sub class of MainFrame
    """

    options_data = OptionsData()
    # This is the save file path name with no extension
    _saved_file: str = C.SAVE_FILE_DIR + options_data.get_lottery_name()

    def __init__(self, parent):
        options_gui.MainFrame.__init__(self, parent)
        self.update_status()

    def update_status(self, text=None) -> None:
        """Utility function used to update the status bar
        :param text: text to display
        :return: None
        """
        status_bar = self.GetStatusBar()
        if text is None:
            opt: OptionsData = self.options_data
            status_bar.SetStatusText(
                f"OPTIONS - Action: "
                f"{opt.get_option_name()}"
                f", Type: "
                f"{opt.get_lottery_name()}"
                f", Lines: "
                f"{opt.number_of_lines}"
            )
        else:
            status_bar.SetStatusText(text)

    def on_exit(self, event):
        """Close the frame and terminate the application."""
        self.Close(force=True)
        event.Skip()

    def on_about(self, _):
        about_info = wx.adv.AboutDialogInfo()
        about_info.SetName(C.PROGRAM)
        about_info.SetVersion(C.VERSION)
        about_info.SetDescription("Lottery numbers generator with wxPython")
        about_info.SetCopyright("(C) 2019-")
        about_info.AddDeveloper(C.AUTHOR)

        wx.adv.AboutBox(about_info)

    def on_type_combo(self, event):
        opt = self.options_data
        opt.lottery_type = event.GetSelection()
        opt.lottery_type_name = C.LOTTERY_CHOICES[opt.lottery_type]
        self._saved_file = C.SAVE_FILE_DIR + opt.lottery_type_name

        self.update_status()
        event.Skip()

    def on_line_control(self, event):
        opt = self.options_data
        opt.number_of_lines = event.GetValue()
        self.update_status()
        event.Skip()

    def on_radiobox(self, event):
        opt = self.options_data
        opt.option = event.GetSelection()
        opt.option_name = C.OPTIONS_CHOICES[opt.option]
        self.update_status()
        event.Skip()

    def on_cancel(self, event):
        self.on_exit(event)

    def on_ok(self, event):
        opt = self.options_data
        if opt.option == 3:
            self.delete_saved_file()
        elif opt.option == 2:
            self.show_saved()
        else:
            self.generate_numbers()
        event.Skip()

    def generate_numbers(self) -> None:
        """Generate several random numbers and optionally save them
        :return:
        """

        def add_leading_zero(values: []) -> []:
            """Add a leading zero to numbers in the list < 10
            :param values: list of numbers
            :return: array containing formatted numbers
            """
            return [f"{v:02d}" for v in values]

        def choose_numbers(maximum: int, quantity: int) -> []:
            """Generate the random numbers required
            :param maximum: the highest number to choose from
            :param quantity: the number of numbers to generate
            :return: a sorted list of generated numbers
            """
            valid_range: range = range(C.RULE_START, maximum)
            return add_leading_zero(sorted(sample(valid_range, quantity)))

        result = ResultsData
        result.clear_data()

        opt = self.options_data
        result.lottery_type_name = opt.get_lottery_name()
        result.number_of_lines = opt.number_of_lines

        main_max, main_qty,\
            extra_max, extra_qty = C.RULES[opt.get_lottery_name()]

        shelf = None
        result.generated = True
        if opt.option == 1:
            msg: str = "The generated numbers have not been saved"
            result.saved = False
            self.update_status(msg)
        else:
            result.saved = True
            directory = Path(C.SAVE_FILE_DIR)
            if not directory.exists():
                directory.mkdir(parents=True)
            shelf = shelve.open(filename=self._saved_file,
                                protocol=C.SHELF_PROTOCOL)
            shelf[C.SHELF_ARGS["DATE"]] = time.time()
            shelf[C.SHELF_ARGS["TYPE"]] = C.LOTTERY_CHOICES[opt.lottery_type]
            shelf[C.SHELF_ARGS["LINES"]] = opt.number_of_lines

        # count the actual number of lines generated
        count: int = 0
        # generate the required number of lines
        for _ in range(opt.number_of_lines):
            # x_1 holds the first group of numbers generated
            # main_max is the the highest number to generate plus 1
            # main_qty is the quantity of numbers to generate each line
            x_1: List[str] = choose_numbers(maximum=main_max,
                                            quantity=main_qty)

            # x_2 holds the secondary group of numbers generated if
            # any are required
            # extra_max is the the highest number to generate plus 1
            # extra_qty is the quantity of numbers to
            # generate in each line
            x_2: List[Union[str, None]] = [None]
            # only generate the second group of numbers if required
            # if extra_qty > 0
            if extra_qty:
                x_2 = choose_numbers(maximum=extra_max, quantity=extra_qty)
            # If shelf is None it means no_save was specified
            if shelf is not None:
                shelf[C.SHELF_ARGS["PART1"] + str(count)] = x_1
                # Save the extra numbers even if there are none to save
                # will return None on subsequent reading
                shelf[C.SHELF_ARGS["PART2"] + str(count)] = x_2
            count += 1
            if x_2 == [None]:
                result.data.append(str(x_1))
            else:
                result.data.append(str(x_1) + " - " + str(x_2))
        # close the save file if there is one
        if opt.get_option_name() != C.OPTIONS_CHOICES[1]:
            shelf.close()
        test: bool = opt.get_option_name() == C.OPTIONS_CHOICES[1]
        msg: str = (
            f"The numbers have{' not' if test else ''}"
            f" been saved and {count} lines were generated"
        )
        self.update_status(msg)
        frm = ResultsFrame(None, result)
        frm.Show()

    def delete_saved_file(self) -> None:
        """Delete a previously saved file
        :return: None
        """
        # add the filename extension
        file_name: str = self._saved_file + C.SAVE_FILE_TYPE
        path = Path(file_name)
        file_exists: bool = path.exists()
        if file_exists:
            path.unlink()
        msg: str = f"File: <{file_name}> was "\
            f"{'deleted' if file_exists else 'not found'}"
        self.update_status(msg)

    def show_saved(self) -> None:
        """Display a previously generated and saved batch of numbers
        :return: None
        """
        opt = self.options_data
        # add the filename extension
        path = Path(self._saved_file + C.SAVE_FILE_TYPE)
        if path.exists() and path.is_file():
            shelf = shelve.open(
                filename=self._saved_file,
                flag=C.SHELF_READONLY,
                protocol=C.SHELF_PROTOCOL,
            )
            save_time = time.localtime(shelf[C.SHELF_ARGS["DATE"]])
            result = ResultsData
            result.clear_data()

            result.saved = False
            result.generated = False
            result.lottery_type_name = opt.get_lottery_name()
            result.stored_date = time.strftime(C.DATE_FORMAT, save_time)
            count: int = 0
            for line in range(shelf[C.SHELF_ARGS["LINES"]]):
                # Don't display second group if none exist
                x_1: List[str] = shelf[C.SHELF_ARGS["PART1"] + str(line)]
                x_2: List[Union[str, None]] = shelf[C.SHELF_ARGS["PART2"]
                                                    + str(line)]
                if x_2 == [None]:
                    result.set_data_item(str(x_1))
                else:
                    result.set_data_item(str(x_1) + " - " + str(x_2))
                count += 1
            result.number_of_lines = count
            shelf.close()
            frm = ResultsFrame(None, result)
            frm.Show()
        else:
            msg: str = f"File <{path}> is missing"
            self.update_status(msg)