#MAD PYTHON CODE:
from copy import deepcopy
from random import shuffle
import random
#example encoding
#      W O R D
#      # R # I
#      # # # S
#      # A S H
#
# list of words: word or ash dish
# marker = 2 (word[0] and word[1] will be horizontal and word[2] and word[3] vertical)
# take all words in order, genome is this order in the individual (ex: 1 3 2 4 = word ash or dish);
# try and assemble the board, check for mismatches, establish fitness, retry

#genotype - order of words
#listOfWords - self-explanatory
class Individual:

    def createBoard(self):
        filledBoard = self.board.copy()
        listOfSpacesCt = 0

        #horizontal words
        for ct in (self.genotype):
            if(listOfSpacesCt == self.splitMark):
                break
            space = self.listOfSpaces[listOfSpacesCt]
            initialX = space[1][1]
            initialY = space[1][0]
            word = self.listOfWords[ct]
            #how many letters we will fill = n
            n = len(word)
            #if the word is going across the borders - fitness tax
            if initialX + len(word) > len(self.board):
                self.fitness +=  (initialX + len(word) - len(self.board)) * self.penaltyQuantifier
                n -=  initialX + len(word) - len(self.board)
            for letterNr in range(0,n):
                #if a letter is on a black space
                if self.board[initialY][initialX + letterNr] == '1' :
                    self.fitness +=  (n - letterNr)
                    break
                else:
                    # if the letter does not match
                    if self.board[initialY][initialX + letterNr] != '0' and self.board[initialY ][initialX + letterNr] != word[letterNr]:
                        self.fitness += 1
                        self.board[initialY][initialX + letterNr] = word[letterNr]
                    #the letter occupies a valid space
                    else:
                        self.board[initialY][initialX + letterNr] = word[letterNr]
            listOfSpacesCt += 1

        #vertical words
        for ct2 in range(listOfSpacesCt, len(self.listOfWords)):
            ct = self.genotype[ct2]
            space = self.listOfSpaces[listOfSpacesCt]
            initialX = space[1][1]
            initialY = space[1][0]
            word = self.listOfWords[ct]
            # how many letters we will fill = n
            n = len(word)
            # if the word is going across the borders - fitness tax
            if initialY + len(word) > len(self.board):
                self.fitness += (initialY + len(word) - len(self.board)) * self.penaltyQuantifier
                n -= initialY + len(word) - len(self.board)
            for letterNr in range(0, n):
                # if a letter is on a black space
                if self.board[initialY  + letterNr][initialX] == '1':
                    self.fitness += (n - letterNr)
                    break
                else:
                    # if the letter does not match
                    if self.board[initialY + letterNr][initialX] != '0' and self.board[initialY + letterNr][initialX] != word[letterNr]:
                        self.fitness += 1
                    # the letter occupies a valid space
                    else:
                        self.board[initialY + letterNr][initialX ] = word[letterNr]
            listOfSpacesCt += 1

        for ct1 in range(0,len(self.board)):
            for ct2 in range(0,len(self.board[0])):
                if self.board[ct1][ct2] == '0':
                    self.fitness += self.penaltyQuantifier

    def mutate(self,probability):
        for ct in range(0,len(self.genotype)):
            if random.uniform(0, 1) < probability:
                positionToSwitch = random.randint(0,len(self.genotype) - 1)
                auxGene = self.genotype[ct]
                self.genotype[ct] = self.genotype[positionToSwitch]
                self.genotype[positionToSwitch] = auxGene
                break

    def finalMutate(self,probability):
        if random.uniform(0, 1) < probability:
            positionToSwitch1 = random.randint(0, len(self.genotype) - 1)
            positionToSwitch2 = random.randint(0, len(self.genotype) - 1)
            while len(self.listOfWords[self.genotype[positionToSwitch1]]) != len(self.listOfWords[self.genotype[positionToSwitch2]]):
                positionToSwitch1 = random.randint(0, len(self.genotype) - 1)
            auxGene = self.genotype[positionToSwitch1]
            self.genotype[positionToSwitch1] = self.genotype[positionToSwitch2]
            self.genotype[positionToSwitch2] = auxGene

    #REFACTOR NAMES WHEN COMPLETELY SOBER
    def cycleCrossover(self, mom, probability):
        cycledGene = []
        selfChild = list(range(0,len(self.genotype)))
        otherChild = []
        for ct in range(0,len(self.genotype)):
            if random.uniform(0, 1) < probability:
                ogGene = self.genotype[ct]
                selfChild[ct] = ogGene
                cycledGene = mom.genotype[ct]
                while(str(ogGene) != str(cycledGene)):
                    for gene in range(0,len(self.genotype)):
                        if self.genotype[gene] == cycledGene:
                            cycledGeneCt = gene
                            selfChild[gene] = self.genotype[gene]
                            cycledGene = mom.genotype[gene]
                            #print(str(ogGene) + " " + str(cycledGene))
            break;
        for ct in range(0, len(self.genotype)):
            if selfChild[ct] != self.genotype[ct]:
                self.genotype[ct] = mom.genotype[ct]
        self.genotype = selfChild

    #that's how the child is made kek
    def crossover(self,mom,dad):
        for ct in range(0,len(mom.genotype)):
            if(random.randint(0,mom.fitness + dad.fitness) > mom.fitness):
                chosenGene = dad.genotype[ct]
            else:
                chosenGene = mom.genotype[ct]
            #find the gene
            for ct1 in range(0,len(self.genotype)):
                #put it in this place to ressemble the parent genotype
                if chosenGene == self.genotype[ct1]:
                    aux = self.genotype[ct]
                    self.genotype[ct] = chosenGene
                    self.genotype[ct1] = aux

    def evalFitness(self):
        self.fitness = 0
        self.createBoard()
        return self.fitness

    def __init__(self, listOfWords, genotype, board, listOfSpaces, splitMark):
        self.fitness = 0
        self.listOfWords = listOfWords
        #order of words
        self.penaltyQuantifier = 3
        self.genotype = deepcopy(genotype)
        self.board = deepcopy(board)
        self.listOfSpaces = listOfSpaces
        self.splitMark = splitMark

    def __str__(self):
        print("GENOTYPE: " + str(self.genotype))
        for line in self.board:
            string = ""
            for char in line:
                string += char
                string = string.replace('1', '*')

            print(string)
        print(">>>>>>>>>>>>>>>>>>")
        print(self.fitness)
        return ""

class Population:
    def __init__(self, individuals):
        self.listOfIndividuals = individuals

class Problem:
    #here we should have black spaces and word list

    def readParams(self,filename):
        try:
            file = open(filename, 'r')
        except IOError:
            print("FAILED READING THE FILE")
            return
        params = file.readline()
        params = params.split(' ')
        self.mutationFactor = float(params[0])
        self.crossoverFactor = float(params[1])
        self.populationSize = int(params[2])
        self.noGenerations = int(params[3])
        print("READ PARAMS SUCCESSFULLY")

    def readData(self, filename):
        try:
            file = open(filename, 'r')
        except IOError:
            print("FAILED READING THE FILE")
            return
        size = file.readline()
        size = size.split(" ")
        self.boardX = int(size[0])
        self.boardY = int(size[1])
        self.board = []
        for counter in range (0,self.boardX):
            line = file.readline()
            self.board.append(list(line[:-1]))
        self.words = file.readline()
        self.words = self.words.split(" ")
        print("READ DATA SUCCESSFULLY")

    def __str__ (self):
        result = ""
        result += "BOARD SIZE:" + str(self.boardX) + " " + str(self.boardY) + "\n"
        result += "BOARD:\n"
        for line in self.board:
            result += str(line) + "\n"
        result += "WORDS:\n"
        result += str(self.words) + "\n"
        return result

    def initPopulation(self):
        for ct in range(0,self.populationSize):
            genome = list(range(0, len(self.words)))
            shuffle(genome)
            newIndividual = Individual(self.words,genome.copy(),deepcopy(self.board),self.listOfSpaces,self.splitmark)
            newIndividual.evalFitness()
            self.population.append(newIndividual)
            print("CREATING INDIVIDUAL #" + str(ct) + " WITH " + str(newIndividual.fitness) + " FITNESS")
        self.population = Population(self.population)

    # here we process the black spots, words and board to find how we can split the
    # list of words in horizontal and vertical word lists
    def processBoard(self):
        print("PROCESSING BOARD")

        minimumWordLength = 100 #arbitrary
        for word in self.words:
            if len(word) < minimumWordLength:
                minimumWordLength = len(word)
        #this will contain a list of numbers of spaces corresponding to each word space on the board

        listOfSpaces = []
        for ct1 in range (0,self.boardY):
            nrOfSpaces = 0
            auxX = -1
            auxY = -1
            for ct2 in range (0,self.boardX):
                if self.board[ct1][ct2] == '0' :
                    nrOfSpaces += 1
                    if( auxY + auxX == -2):
                        auxY = ct1
                        auxX = ct2
                if self.board[ct1][ct2] == '1':
                    if nrOfSpaces >= minimumWordLength:
                        listOfSpaces.append([nrOfSpaces,[auxY,auxX]])
                    nrOfSpaces = 0
                    auxY = -1
                    auxX = -1

            if nrOfSpaces >= minimumWordLength:
                listOfSpaces.append([nrOfSpaces,[auxY,auxX]])

        #the vertical words start from here
        self.splitmark = len(listOfSpaces)
        for ct1 in range (0,self.boardX):
            nrOfSpaces = 0
            auxX = -1
            auxY = -1
            for ct2 in range(0, self.boardY):
                if self.board[ct2][ct1] == '0':
                    nrOfSpaces += 1
                    if (auxY + auxX == -2):
                        auxY = ct1
                        auxX = ct2
                if self.board[ct2][ct1] == '1':
                    if nrOfSpaces >= minimumWordLength:
                        listOfSpaces.append([nrOfSpaces,[auxX,auxY]])
                    nrOfSpaces = 0
                    auxY = -1
                    auxX = -1
            if nrOfSpaces >= minimumWordLength:
                listOfSpaces.append([nrOfSpaces,[auxX,auxY]])
        self.listOfSpaces = listOfSpaces
        print("FINISHED PROCESSING BOARD")
        print(self.listOfSpaces)

    def __init__(self, dataFilename, paramsFilename):
        print("INITIALISING PROBLEM FROM FILES: " + dataFilename + ", " + paramsFilename)
        self.population = []
        self.splitmark = -1
        self.readData(dataFilename)
        self.readParams(paramsFilename)
        self.processBoard()
        print(self)
        self.initPopulation()
        print("FINISHED INITIALISING PROBLEM")

class EvolutiveAlgorithm:

    def statistics(self):
        bestFitness = self.population.listOfIndividuals[0].fitness
        bestIndividual = -1
        avarageFitness = 0
        for individual in self.population.listOfIndividuals:
            avarageFitness += individual.fitness
            if individual.fitness <= bestFitness:
                bestFitness = self.population.listOfIndividuals[0].fitness
                bestIndividual = deepcopy(individual)
        avarageFitness = float(avarageFitness) / self.populationSize
        return [bestFitness,deepcopy(bestIndividual),avarageFitness]

    def run(self):
        bestFitness = 500
        bestIndividual = []
        for ct in range(0,self.problem.noGenerations):
            self.iteration()
            pair = self.statistics()
            if pair[0] < bestFitness:
                bestFitness = pair[0]
                bestIndividual = pair[1]
                print("ITERATION #" + str(ct) + " FITNESS " + str(pair[0]) + " AVERAGE: " + str(pair[2]))
            if ct % 100 == 0:
                print("ITERATION #" + str(ct) + " FITNESS " + str(pair[0]) + " AVERAGE: " + str(pair[2]))
        print("BEST INDIVIDUAL: ")
        if(len(set(bestIndividual.genotype)) == len(bestIndividual.genotype)):
            print(bestIndividual)

    def iteration(self):
        newPopulation = []
        shuffle(self.population.listOfIndividuals)
        for ct in range(0,self.populationSize,2):
            parent1 = self.population.listOfIndividuals[ct]
            parent2 = self.population.listOfIndividuals[ct+1]
            genome = list(range(0, len(self.problem.words)))
            #newIndividual = deepcopy(parent1)
            #newIndividual.cycleCrossover(parent2,self.problem.crossoverFactor)
            newIndividual = Individual(self.problem.words, genome, self.problem.board, self.problem.listOfSpaces, self.splitMark)
            newIndividual.crossover(parent1, parent2)
            newIndividual.mutate(self.problem.mutationFactor)
            newIndividual.evalFitness()
            if newIndividual.fitness <= 5:
                newIndividual.finalMutate(self.problem.mutationFactor)
                newIndividual.evalFitness()
            newPopulation.append(newIndividual)
        for kid in newPopulation:
            self.population.listOfIndividuals.append(kid)
        self.population.listOfIndividuals.sort(key=lambda x: x.fitness, reverse=False)
        self.population.listOfIndividuals = self.population.listOfIndividuals[:self.populationSize]

    def __init__(self, problem):
        self.problem = problem
        self.splitMark = problem.splitmark
        self.population = problem.population
        self.populationSize = problem.populationSize

myProblem = Problem("easyData.in","params.in")
myAlgo = EvolutiveAlgorithm(myProblem)
print("PROCEED WITH ITERATIONS?")
x = "yes"
while x != "no":
    x = input()
    x = x.split(" ")
    if(len(x) > 1):
        myProblem.mutationFactor = float(x[1])
    print("MUTATION FACTOR:" + str(myProblem.mutationFactor))
    myAlgo.run()

#print(myProblem)


import os
from os import listdir

templates = ['404.html', 'about.html', 'addnote.html', 'base.html', 'editnote.html', 'edituser.html', 'index.html',
            'login.html', 'notes.html', 'register.html', 'singlenote.html', 'user.html']
important_files_folders = ['__init__.py', 'errors.py', 'forms.py', 'models.py', 'routes.py']

def check_pages_exist():
    real_templates = os.listdir('./app/templates')
    for template in templates:
        assert(template in real_templates)

def check_files_exist():
    app_files = os.listdir('./app')
    for file in important_files_folders:
        assert(file in app_files)

def check_files_do_not_exist():
    all_files = os.listdir('./')
    assert(not '.env' in all_files)
    assert(not 'app.db' in all_files)

def check_information_hidden():
    with open ('.flaskenv', 'r') as myfile:
        env = myfile.readlines()
        assert(not 'SECRET_KEY' in env)
        assert(not 'MAIL_PASSWORD' in env)
    myfile.close()

check_pages_exist()
check_files_exist()
check_files_do_not_exist()
check_information_hidden()


import unittest
import subprocess
from time import sleep
import utils.file_utils as fu
import crawler.ffmod as ffm
import sttest
SYS_CALLS = ["open", "read", "write"]

class Test(sttest.STTest):
        def test_simple_trace(self):
            sys_log_file = open(self.vi.sys_log, "w")
            subprocess.Popen("strace ls",
                             stderr=sys_log_file, stdout=sys_log_file,
                             shell=True)
            sys_log_file.close()
            sleep(5)  # wait for strace output to be available
            sys_logs = fu.read_file(self.vi.sys_log)
            for sys_call in SYS_CALLS:
                self.assertIn(sys_call, sys_logs)

        def test_attach_trace(self):
            sys_log_file = open(self.vi.sys_log, "w")
            vdisplay = ffm.start_xvfb()
            driver, _, proc = ffm.get_browser(self.vi.ff_log)
            subprocess.Popen("strace -p %d" % proc.pid,
                             stderr=sys_log_file, stdout=sys_log_file,
                             shell=True)
            driver.get("https://google.com")
            sleep(1)
            driver.close()
            proc.terminate()
            driver.quit()
            ffm.stop_xvfb(vdisplay)
            sys_log_file.close()
            sleep(5)  # wait for strace output to be available
            sys_logs = fu.read_file(self.vi.sys_log)
            for sys_call in SYS_CALLS:
                self.assertIn(sys_call, sys_logs,
                              """Cannot trace process, make sure you set up
                    ptrace permissions in /proc/sys/kernel/yama/ptrace_scope as
                    described in etc/setup.sh.""")

if __name__ == "__main__":
    unittest.main()


from frappe.model.document import Document

class BulkTransactionLog(Document):
	def validate(self):
		status = 'Completed'
		for i in self.bulk_transaction_log_table:
			if i.status != "Completed":
				status = 'Pending'
				break
		self.transaction_status = status


import math
import logging
import torch
import torch.nn as nn
from torch.nn import functional as F
logger = logging.getLogger(__name__)

########################################################################################################
# MHA: Multi-head Attention + Rotary Encoding
########################################################################################################

class RotaryEmbedding(torch.nn.Module):
    def __init__(self, dim, base=10000):
        super().__init__()
        inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim))
        self.register_buffer('inv_freq', inv_freq)
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None
    def forward(self, x, seq_len=None):
        if seq_len != self.seq_len_cached:
            self.seq_len_cached = seq_len
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.einsum('i,j->ij', t, self.inv_freq)
            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
            self.cos_cached = emb.cos()
            self.sin_cached = emb.sin()
        return self.cos_cached, self.sin_cached
def rotate_half(x):
    x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
    return torch.cat((-x2, x1), -1)
@torch.jit.script
def apply_rotary_pos_emb(q, k, cos, sin):
    cos, sin = cos[...,:q.shape[-2],:], sin[...,:q.shape[-2],:]
    return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)

class MHA(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        assert config.n_attn % config.n_head == 0
        self.n_head = config.n_head
        self.ctx_len = config.ctx_len
        self.head_size = config.n_attn // config.n_head

        self.query = nn.Linear(config.n_embd, config.n_attn)
        self.key = nn.Linear(config.n_embd, config.n_attn)
        self.value = nn.Linear(config.n_embd, config.n_attn)
        self.register_buffer("mask", torch.tril(torch.ones(config.ctx_len, config.ctx_len)))

        if self.config.ROTARY_POS_EMB:
            self.rotary_ndims = int(self.head_size * 0.5)
            self.rotary_emb = RotaryEmbedding(self.rotary_ndims)

        self.output = nn.Linear(config.n_attn, config.n_embd)

    def forward(self, x):
        B, T, C = x.size()

        q = self.query(x).view(B, T, self.n_head, self.head_size).transpose(1, 2)       # (B, T, C) -> (B, nh, T, hs)
        k = self.key(x).view(B, T, self.n_head, self.head_size).transpose(1, 2)         # (B, T, C) -> (B, nh, T, hs)
        v = self.value(x).view(B, T, self.n_head, self.head_size).transpose(1, 2)       # (B, T, C) -> (B, nh, T, hs)

        if self.config.ROTARY_POS_EMB:
            q, query_pass = q[..., :self.rotary_ndims], q[..., self.rotary_ndims:]
            k, key_pass = k[..., :self.rotary_ndims], k[..., self.rotary_ndims:]
            cos, sin = self.rotary_emb(q, seq_len=T)
            q, k = apply_rotary_pos_emb(q, k, cos, sin)                                     # rotary encoding
            q = torch.cat((q, query_pass), dim=-1)
            k = torch.cat((k, key_pass), dim=-1)

        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))                 # self-attention: (B, nh, T, hs) * (B, nh, hs, T) -> (B, nh, T, T)
        att = att.masked_fill(self.mask[:T,:T] == 0, float('-inf'))                     # causal mask
        att = F.softmax(att, dim = -1)                                                  # softmax

        x = att @ v                                                                     # (B, nh, T, T) * (B, nh, T, hs) -> (B, nh, T, hs)
        x = x.transpose(1, 2).contiguous().view(B, T, -1)                               # (B, nh, T, hs) -> (B, T, nh, hs) -> (B, T, C)

        x = self.output(x)
        return x

class GeGLU(torch.nn.Module):
    def __init__(self, config):
        super().__init__()
        hidden_sz = 3 * config.n_ffn
        self.key = nn.Linear(config.n_embd, hidden_sz)
        self.value = nn.Linear(config.n_embd, hidden_sz)
        self.weight = nn.Linear(hidden_sz, config.n_embd)

    def forward(self, x):
        B, T, C = x.size()
        k = self.key(x)
        v = self.value(x)
        y = self.weight(F.gelu(k) * v)
        return y

########################################################################################################
# The GPT Model with our blocks
########################################################################################################

class GPTConfig:
    def __init__(self, vocab_size, ctx_len, **kwargs):
        self.vocab_size = vocab_size
        self.ctx_len = ctx_len
        for k,v in kwargs.items():
            setattr(self, k, v)

class Block(nn.Module):
    def __init__(self, config, layer_id):
        super().__init__()
        self.config = config
        self.layer_id = layer_id

        if (not self.config.USE_POST_LN) and (self.config.USE_SMALL_EMB) and (self.layer_id == 0): # LN(SmallInit(Emb))
            self.lnPre = nn.LayerNorm(config.n_embd)

        self.ln1 = nn.LayerNorm(config.n_embd)
        self.ln2 = nn.LayerNorm(config.n_embd)

        self.att = MHA(config)
        self.ffn = GeGLU(config)

    def forward(self, x):
        if self.config.USE_POST_LN:
            x = self.ln1(x)
            x = x + self.att(x)
            x = self.ln2(x)
            x = x + self.ffn(x)
        else:
            if self.config.USE_SMALL_EMB and self.layer_id == 0: # LN(SmallInit(Emb))
                x = self.lnPre(x)

            x = x + self.att(self.ln1(x))
            x = x + self.ffn(self.ln2(x))

        return x

class GPT(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.ctx_len = config.ctx_len

        self.emb = nn.Embedding(config.vocab_size, config.n_embd)
        if not self.config.ROTARY_POS_EMB:
            self.pos_emb = nn.Parameter(torch.zeros(1, config.ctx_len, config.n_embd)) # note: i initialize abs.pos.emb to zero

        self.blocks = nn.Sequential(*[Block(config, i) for i in range(config.n_layer)])
        self.ln_out = nn.LayerNorm(config.n_embd)
        self.head = nn.Linear(config.n_embd, config.vocab_size, bias=False)

        self.apply(self._init_weights)
        logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters()))

    def _init_weights(self, module):

        if isinstance(module, (nn.Linear)):
            module.weight.data.normal_(mean=0.0, std=0.02)

        if isinstance(module, (nn.Embedding)):
            if self.config.USE_SMALL_EMB:
                nn.init.uniform_(module.weight, a=-1e-4, b=1e-4) # SmallInit(Emb)
            else:
                module.weight.data.normal_(mean=0.0, std=0.02)

        if isinstance(module, nn.Linear) and module.bias is not None:
            module.bias.data.zero_()

    def configure_optimizers(self, train_config):
        decay = set() # separate out all parameters to those that will and won't experience regularizing weight decay
        no_decay = set()
        whitelist_weight_modules = (nn.Linear, )
        blacklist_weight_modules = (nn.LayerNorm, nn.Embedding)
        for mn, m in self.named_modules():
            for pn, p in m.named_parameters():
                fpn = '%s.%s' % (mn, pn) if mn else pn # full param name
                if pn.endswith('bias') or ('time' in fpn) or ('head' in fpn) or ('scale' in fpn) or ('pos_emb' in fpn):
                    no_decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules):
                    decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):
                    no_decay.add(fpn)
        param_dict = {pn: p for pn, p in self.named_parameters()} # validate that we considered every parameter
        inter_params = decay & no_decay
        union_params = decay | no_decay
        assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), )
        assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \
                                                    % (str(param_dict.keys() - union_params), )
        optim_groups = [
            {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": train_config.weight_decay},
            {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0},
        ]
        optimizer = torch.optim.AdamW(optim_groups, lr=train_config.learning_rate, betas=train_config.betas, eps=train_config.eps)
        return optimizer

    def forward(self, idx, targets=None):
        B, T = idx.size()
        assert T <= self.ctx_len, "Cannot forward, because len(input) > model ctx_len."

        x = self.emb(idx)
        if not self.config.ROTARY_POS_EMB:
            x = x + self.pos_emb[:, :T, :]

        x = self.blocks(x)

        x = self.ln_out(x)

        x = self.head(x)

        # print(self.emb.weight.detach().cpu().numpy()) # <-------- Show embedding matrix changes

        loss = None
        if targets is not None:
            loss = F.cross_entropy(x.view(-1, x.size(-1)), targets.view(-1))
        return x, loss


import pandas as pd
import numpy as np
import ftapp.core.constants as constants
import pickle
import time
import matplotlib.pyplot as plt
import io
import base64
timestamp = time.strftime('%Y%m%d%H%M')
import datetime as dt
import os

from flask import Flask

from ftapp import logger
import seaborn as sns
from ftapp import db
from factor_analyzer import FactorAnalyzer
import traceback
from sklearn.model_selection import GridSearchCV
from yellowbrick.classifier import ClassificationReport

#models
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import scale
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix

from sklearn.model_selection import train_test_split
#train test split
from definitions import ROOT_DIR

def split_data(x,y, split):
    #remove FB from the list
    fb_idx = x.index.get_loc('FB')
    fb_x = x.loc['FB']
    x.drop(['FB'])
    print("y",y)
    print("fb_idx", fb_idx)
    fb_y = y.iloc[fb_idx]
    y.drop([fb_idx])
    X_train, X_test, y_train, y_test, idx_train, idx_test = train_test_split(x, y, x.index, test_size=split)

    X_test = X_test.append(fb_x)
    y_test = y_test.append(fb_y)
    print("idx_test", idx_test)
    idx_test = idx_test.append(pd.Index(['FB']))
    print("idx_test")
    for each in idx_test:
        if each =='FB':
            print("YES ADDED FB to idx")
    for each in X_test.index:
        if each =='FB':
            print("YES ADDED FB to test")

    return X_train, X_test, y_train, y_test, idx_train, idx_test

def get_data(symbols, dates, addSPY=True, colname = constants.CLOSE_COLUMN):
    '''

    :param symbols:
    :param dates:
    :param addSPY:
    :param colname:
    :return: data df
    '''

    df = pd.DataFrame(index=dates)
    # add SPY if not already in symbols
    if addSPY and 'SPY' not in symbols:
        symbols = ['SPY'] + symbols

        # loop over all the stocks in the list and add to the column
    for symbol in symbols:
        file_name = os.path.join(ROOT_DIR, constants.MARKETDATA_FOLDER, symbol+'.csv')

        df_temp = pd.read_csv(file_name, index_col='Date',
                              parse_dates=True, usecols=['Date', colname], na_values=['nan'])
        df_temp = df_temp.rename(columns={colname: symbol})
        df = df.join(df_temp)
        # filter out the days where SPY is not available - non-trading days
        if symbol == 'SPY':
            df = df.dropna(subset=["SPY"])

    return df

# helper function to compute the portfolio values
def compute_portvals(orders, start_date, end_date, start_val=constants.SV, commission=constants.COMMISSION, impact=constants.IMPACT, ):

    orders = orders.sort_index()
    stock_list = list(set(orders["Symbol"]))

    print("portval start, end", start_date, end_date)

    dates = pd.date_range(start_date, end_date)

    price = get_data(stock_list, dates)

    price.fillna(method='ffill', inplace=True)
    price.fillna(method='bfill', inplace=True)

    price["cash"] = 1.0

    trades = pd.DataFrame(np.zeros((price.shape)), price.index, price.columns)

    mask = (orders['Order'] == "SELL")
    orders['Shares'] = orders['Shares'].mask(mask, -1 * orders["Shares"])

    for i, row in orders.iterrows():
        try:
            trades.loc[i, row["Symbol"]] = trades.loc[i, row["Symbol"]] + row["Shares"]
            trades.loc[i, 'cash'] = trades.loc[i, 'cash'] - row["Shares"] * price.loc[
                i, row["Symbol"]] - commission - abs(row["Shares"]) * price.loc[i, row["Symbol"]] * impact
        except:
            pass

    holdings = pd.DataFrame(np.zeros((price.shape)), price.index, price.columns)
    print(holdings)

    holdings.iloc[0, :-1] = trades.iloc[0, :-1].copy()
    holdings.iloc[0, -1] = trades.iloc[0, -1] + start_val

    for i in range(1, len(trades.index.values)):
        holdings.iloc[i] = holdings.iloc[i - 1] + trades.iloc[i]
    values = holdings * price

    values['portval'] = values.sum(axis=1)
    portval = values['portval']

    return portval

def generate_data(df_trades, symbol, start, end, sv, commission, impact, mode):
    # run model and generate data for graph
    '''
    Return:
    - portvals
    - trades
    - benchmark

    '''
    print(df_trades)
    df_b = df_trades.loc[df_trades.iloc[:, 0] > 0]
    if len(df_b) > 0:
        df_b.loc[:, 'Order'] = "Buy"

    df_s = df_trades.loc[df_trades['Shares'] < 0]
    if len(df_s) > 0:
        df_s.loc[:, 'Order'] = "Sell"
    trades = pd.concat([df_b, df_s])
    trades = trades.sort_index()

    portvals = compute_portvals(trades, start, end, start_val=sv, commission=commission, impact=impact)
    print(portvals)

    if isinstance(portvals, pd.DataFrame):
        portvals_test = portvals[portvals.columns[0]]  # just get the first column
    else:
        "warning, code did not return a DataFrame"

    dates = pd.date_range(start, end)

    # print portvals
    price = get_data([symbol], dates)[symbol]

    benchmark = pd.DataFrame(0, portvals.index, columns=["Benchmark_Return"])
    cash = sv - price[0] * 1000
    benchmark = cash + price * 1000
    benchmark[0] = sv

    #

    # save portfolio value history to disk
    with open('{}/{}-{}.p'.format(constants.PORTVAL_FOLDER,timestamp, mode), 'wb') as fp:
        pickle.dump(portvals, fp)

    print(portvals)
    return portvals, benchmark

#helper function to get the stats
def compute_stat(prices, rfr = constants.RFR):
    daily_rets = (prices/prices.shift(1))-1
    #print portvals
    #print portvals.shift(1)
    daily_rets = daily_rets[1:]
    cr = (prices[-1]/prices[0])-1
    adr = daily_rets.mean()
    sddr = daily_rets.std()
    sr = np.sqrt(252) * (daily_rets - rfr).mean() / sddr

    return cr, adr, sddr, sr

def get_IPO_dates(ticker):
    '''

    :param ticker: given ticker
    :return: str dates
    '''
    df = pd.read_csv('{}/{}.csv'.format(constants.USERDATA_FOLDER, constants.IPO_DATA), index_col = 0)

    ticker_dates = df.loc[ticker]
    ipo = dt.datetime.strptime(ticker_dates[0], '%m/%d/%Y')
    lockup = dt.datetime.strptime(ticker_dates[1], '%m/%d/%Y')
    return ipo, lockup

def print_stats(portvals, benchmark,name,start, end, rfr = constants.RFR):

    #compute stats
    cum_ret, avg_daily_ret, std_daily_ret, sharpe_ratio = compute_stat(portvals, rfr=0.0)
    cum_ret_SPY, avg_daily_ret_SPY, std_daily_ret_SPY, sharpe_ratio_SPY = compute_stat(benchmark, rfr=0.0)

    #print out results
    print ("%s Dataset"%(name))
    print ("Date Range: {} to {}".format(start, end))
    print ()
    print ("Sharpe Ratio of Fund: {}".format(sharpe_ratio))
    print ("Sharpe Ratio of Buy & Hold : {}".format(sharpe_ratio_SPY))
    print()
    print ("Cumulative Return of Fund: {}".format(cum_ret))
    print ("Cumulative Return of Buy & Hold : {}".format(cum_ret_SPY))
    print()
    print ("Standard Deviation of Fund: {}".format(std_daily_ret))
    print ("Standard Deviation of Buy & Hold : {}".format(std_daily_ret_SPY))
    print()
    print ("Average Daily Return of Fund: {}".format(avg_daily_ret))
    print ("Average Daily Return of Buy & Hold : {}".format(avg_daily_ret_SPY))
    print()
    print ("Final Portfolio Value: {}".format(portvals[-1]))
    return None

app = Flask(__name__)
UPLOAD_FOLDER = './user_data'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
def load_screener_data( inputFile):

    log_info = f'Start to upload screening factors with {inputFile}'
    print(log_info)
    print(log_info)
    #filename = f'{constants.USERINPUT_FOLDER}/{inputFile}'
    try:
        filename =os.path.join(app.config['UPLOAD_FOLDER'],inputFile)
        print("filename", filename)
        input = pd.read_csv(filename, index_col=0)
        print("input", input.head())
        print("load data success")
        '''if y_col ==-1:
            y = input[y_col]
            buy_info = BuyInfo.query.filter(BuyInfo.indicator_id==indicator_id, BuyInfo.symbol==symbol, BuyInfo.buydate==dbDate).first()
            if (not buy_info and math.isnan(float(buyprice)) == False and buyprice > 0.0001):
                buyinfo = BuyInfo(buydate=dbDate, symbol=symbol, buyprice=buyprice, openprice=openprice, closeprice=closeprice, highprice=highprice, lowprice=lowprice, indicator_id=indicator_id)
                db.session.add(buyinfo)

        db.session.commit()
        logger.info('Finish to upload buy indicator')'''
        return input

    except Exception:
        print("Exception for initialing ")
        logger.exception(f'Exceptiion for initialing test')
        traceback.print_exc()

def clean_data(input, cat_cols, num_cols):

    #sring split
    print("cat cols", cat_cols)
    print("num_cols", num_cols)
    if cat_cols =='0':
        pass
    elif cat_cols == '-1':
        cat_cols = input.columns.drop(num_cols)
        for each in cat_cols:
            print(input[each])
            input[each] = pd.factorize(input[each])[0]
    else:
        x = cat_cols.split(',')
        print(x)
        for each in x:
            print(input[each])
            input[each] = pd.factorize(input[each])[0]

    if num_cols =='0':
        pass
    elif num_cols == '-1':
        num_cols = input.columns.drop(x)
        input[num_cols] = input[num_cols].apply(pd.to_numeric, errors='coerce')
    else:
        num_cols = num_cols.split(',')
        print(num_cols)
        input[num_cols] = input[num_cols].apply(pd.to_numeric, errors='coerce')
    return input

# loadings
def discretize_y(X_tensor, Y_matrix, upper_percentile=60, lower_percentile=40):
    img = io.BytesIO()

    plt.switch_backend('Agg')

    plt.style.use('ggplot')
    n_stocks, n_factors = np.shape(X_tensor)
    print("Y", Y_matrix)
    upper = np.percentile(Y_matrix, upper_percentile)
    lower = np.percentile(Y_matrix, lower_percentile)

    Y_binary = np.zeros(n_stocks)

    for i in range(n_stocks):
        print("value", Y_matrix.iloc[i])
        if Y_matrix.iloc[i] > upper:
            Y_binary[i] = 1
        elif Y_matrix.iloc[i] <= lower:
            Y_binary[i] = -1

    #print(Y_binary)
    return Y_binary

def create_graph(portvals, benchmark, trades_df, name, save = False, show = True):
    #print(benchmark)
    img = io.BytesIO()

    plt.switch_backend('Agg')

    plt.style.use('ggplot')

    b_temp = benchmark / benchmark.iloc[0]
    pv_temp = portvals / portvals.iloc[0]
    plt.plot(b_temp, label='Benchmark (Buy & Hold)', color="g")
    plt.plot(pv_temp, label='Portfolio', color="r")
    buys = trades_df["Shares"] > 0
    sells = trades_df["Shares"] < 0
    print("buys",trades_df.index[trades_df["Shares"] >0].tolist())
    for i in range(len(trades_df)):
        if trades_df.iloc[i]["Shares"] > 0:
            plt.axvline(x=trades_df.index[i],label = 'Buys (Long)', color='blue')
        elif trades_df.iloc[i]["Shares"] < 0:
            plt.axvline(x=trades_df.index[i], label = 'Sells (Short)', color='black')
    name = name
    plt.xlabel('Date')
    plt.xticks(rotation=70)
    plt.ylabel('Normalized Daily Value')
    plt.title('Deep Reinforcement Learner vs Benchmark Performance - ' + str(name))

    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())

    plt.savefig(img, format='png')

    img.seek(0)
    graph_url = base64.b64encode(img.getvalue()).decode()
    plt.close()

    return 'data:image/png;base64,{}'.format(graph_url)

def save(x,y, screener_name):
    x.to_csv('./user_data/{}-x.csv'.format(screener_name))
    pd.DataFrame(y).to_csv('./user_data/{}-y.csv'.format(screener_name))

def eigenvalues_plt(data):
    img = io.BytesIO()

    plt.switch_backend('Agg')

    plt.style.use('ggplot')
    fa = FactorAnalyzer()
    fa.fit(data)
    eigen_values, vectors = fa.get_eigenvalues()
    plt.figure(figsize=(10, 10))
    plt.scatter(range(1, data.shape[1] + 1), eigen_values)
    plt.plot(range(1, data.shape[1] + 1), eigen_values)
    plt.title('Factor Importance by Eigenvalues')

    plt.xlabel('Factors')
    plt.ylabel('Eigenvalue')
    plt.grid()

    plt.savefig(img, format='png')

    img.seek(0)
    graph_url = base64.b64encode(img.getvalue()).decode()
    plt.close()

    return 'data:image/png;base64,{}'.format(graph_url)

def corr_matrix(data):
    img = io.BytesIO()

    plt.switch_backend('Agg')

    plt.style.use('ggplot')
    data.columns = ["V"+str(i) for i in range(1, len(data.columns)+1)]  # rename column names to be similar to R naming convention
    # financials.V1 = financials.V1.astype(str)
    # X = financials  # independent variables data

    corrmat = data.corr()
    plt.figure(figsize=(10, 10))
    sns.heatmap(corrmat, vmax=1., square=False, cmap="YlGnBu").xaxis.tick_top()
    plt.title('Correlation Matrix')
    plt.savefig(img, format='png')

    img.seek(0)
    graph_url = base64.b64encode(img.getvalue()).decode()
    plt.close()

    return 'data:image/png;base64,{}'.format(graph_url)

def mosthighlycorrelated(mydataframe, numtoreport):
    # find the correlations
    cormatrix = mydataframe.corr()
    # set the correlations on the diagonal or lower triangle to zero,
    # so they will not be reported as the highest ones:
    cormatrix *= np.tri(*cormatrix.values.shape, k=-1).T
    # find the top n correlations
    cormatrix = cormatrix.stack()
    cormatrix = cormatrix.reindex(cormatrix.abs().sort_values(ascending=False).index).reset_index()
    # assign human-friendly names
    cormatrix.columns = ["FirstVariable", "SecondVariable", "Correlation"]
    return cormatrix.head(numtoreport)

def standardize(x):
    standardisedX = scale(x)
    standardisedX = pd.DataFrame(standardisedX, index=x.index, columns=x.columns)
    standardisedX.apply(np.mean)
    standardisedX.apply(np.std)
    return standardisedX

def pca_summary(pca, standardised_data):
    names = ["PC"+str(i) for i in range(1, len(pca.explained_variance_ratio_)+1)]
    a = list(np.std(pca.transform(standardised_data), axis=0))
    b = list(pca.explained_variance_ratio_)
    c = [np.sum(pca.explained_variance_ratio_[:i]) for i in range(1, len(pca.explained_variance_ratio_)+1)]
    columns = pd.MultiIndex.from_tuples([("sdev", "Standard deviation"), ("varprop", "Proportion of Variance"), ("cumprop", "Cumulative Proportion")])
    summary = pd.DataFrame(zip(a, b, c), index=names, columns=columns)
    return summary

param_grid = {"Decision Tree": [
  {'n_estimators': [7,8,9,10,11,100,400,1000],
   'algorithm':['SAMME.R','SAMME' ],
   'learning_rate': [0.05,0.1,0.2,0.3, 0.4,0.8,1.0]}
 ],
    "Random Forest": [{'n_estimators': [700, 800, 900],
    'max_features': ['auto', 'sqrt', 'log2'],
    'criterion': ['gini', 'entropy'],
    'min_samples_split':[2,3,4],
    'max_depth':[None,10,20]
}],
    "Ada Boost": [
  {'n_estimators': [7,8,9,10,11,100,400,1000],'algorithm':['SAMME.R','SAMME' ],'learning_rate': [0.05,0.1,0.2,0.3, 0.4,0.8,1.0]}
 ],
    "Neural Network": [
  {'fit_intercept':[True, False],'normalize':[True, False]}
 ]
}

choices=[(1,'Decision Tree'),(2,'Ada Boost'), (3,'Neural Network'), (4,'Random Forest')]

models = [DecisionTreeClassifier(),\
          AdaBoostClassifier(n_estimators=100, learning_rate=0.6, random_state=0),\
          LinearRegression(), \
          RandomForestClassifier(max_features= 'sqrt' ,n_estimators=700, oob_score = True) ]

def evaluate(model, X_train, X_test, y_train, y_test, optimize =False, cv = 5):
    print("model", int(model))
    clf = models[int(model)-1]
    if optimize:
        clf = GridSearchCV(clf, param_grid[model], cv=cv)
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    score = clf.score(X_test, y_test)

    return score, y_pred, clf

def confusion_matrix(classifier, x,y):
    img = io.BytesIO()

    plt.switch_backend('Agg')

    plt.style.use('ggplot')
    plt.figure(figsize=(10, 10))
    print(classifier)

    plot_confusion_matrix(classifier, x, y, cmap='YlGnBu')

    plt.title('Correlation Matrix')
    plt.savefig(img, format='png')

    img.seek(0)
    graph_url = base64.b64encode(img.getvalue()).decode()
    plt.close()

    return 'data:image/png;base64,{}'.format(graph_url)

def classificationreport(clf, classes, X_train, y_train, X_test, y_test):
    #classes = ['increase','little change', 'decrease']
    img = io.BytesIO()

    #plt.switch_backend('Agg')

    #plt.style.use('ggplot')

    visualizer = ClassificationReport(clf, classes=classes, support=True)

    visualizer.fit(X_train, y_train)        # Fit the visualizer and the model
    visualizer.score(X_test, y_test)        # Evaluate the model on the test data
    visualizer.show(outpath = img)                       # Finalize and show the figure
    plt.figure(figsize=(8, 8))

    img.seek(0)
    graph_url = base64.b64encode(img.getvalue()).decode()
    return 'data:image/png;base64,{}'.format(graph_url)

def get_stocks_classification(y, index):
    increase = []
    stay = []
    decrease = []
    for i in range(len(y)):
        if y[i]==-1:
            decrease.append(index[i])
        elif y[i]==1:
            increase.append(index[i])
        else:
            stay.append(index[i])
    return increase, stay, decrease


# stack을 이용한 풀이이지만 해당 문제의 시간 복잡도는 비효율적임
# 다른 풀이도 확인했으나 풀이가 모두 비슷한 것으로 보아 문제 오류로 보임
def solution(s):
    answer = []
    for string in s:
        stack = []
        cnt_110 = 0
        for str_i in string:
            if len(stack) >= 2 and stack[-1] == "1" and stack[-2] == "1" and str_i == "0":
                cnt_110 += 1
                stack.pop()
                stack.pop()
            else:
                stack.append(str_i)

        cnt_1 = 0
        for element in stack[::-1]:
            if element == "0":
                break
            else:
                cnt_1 += 1

        answer.append(''.join(stack[:len(stack)-cnt_1]) + "110"*cnt_110 + ''.join(stack[len(stack)-cnt_1:]))
    return answer

#my solution: 오답
def solution(s):
    result = []
    i = 0
    for string in s:
        cnt = 0
        while True:
            ori_len = len(string)
            string = ''.join(string.split("110"))
            cnt += (ori_len - len(string)) // 3
            if ori_len == len(string):
                break
        idx = string.find("111")
        if idx == -1:
            result.append("110"*cnt + string)
        else:
            result.append(string[:idx] + "110"*cnt + string[idx:])
    return result

print(solution(["1110","100111100","0111111010"]))


"""
    OBJECTIVES:
    - add more nodes that are connected to at least 2 other nodes inside artist_data
    - to mine more neighbors of nodes from IndieNesia playlist (since there are not enough data/imbalanced)
    - get rid or avoid mining nodes with degree == 1

    TO-DO:
    - get rids nodes with deg==1:
        - for existings(nodes degree=1):
            - delete (preferred in top hits playlist)
            - find existing neighbors in dump (preferred in IndieNesia playlist)
        - for new:
            - check if nodes (artist_id) is connected with the in_traverse
              && at least the other one* inside artist_data
              *(listed in neighbors of all existing data (artist_data))
              [YES]--> add to queue

    STEPS
    [LOCAL]
    - read from local data, both in dump and artist_data.
    - get collaborators of indienesia (with no exclusion), only in dump.
    - filter out gotten collaborators ==> list_of_in_dump.
    - create intersection of set collaborators of each in list_of_in_dump...
    - ... with everyone in artist_data.
    - [one transaction with DB] Add to traverse queue if len intersection >= 2

    [DB CONNECTION]
    - after queue creation complete
    - move all collaborators from dump (id in queue) to artist_data
"""
import json

import time
from tqdm import tqdm
from google.cloud import firestore
import os

os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = './credentials/spotify-co-featuring2gra-34f8c-firebase-adminsdk-k0pe2-bfcd86823f.json'

db = firestore.Client(
    project="spotify-co-featuring2gra-34f8c"
)

DUMP_PATH = 'backup/data/export/staging-exceeds_dump_v2.json'
ARTIST_DATA_PATH = 'backup/data/export/staging-artist_data_v2.json'

def load_ids(filepath):
    excluded_ids = []
    with open(filepath) as json_file:
        data = json.load(json_file)
        for x in data['collections']:
            excluded_ids.append(x)
    return excluded_ids

def load_data(filepath):
    data = {}
    with open(filepath) as json_file:
        data = json.load(json_file)
    return data

def indienesia_only(data):
    collections = data['collections']
    out = {'collections':{}}
    for artist_id, payload in collections.items():
        if payload['playlist'] == "IndieNesia":
            out['collections'][artist_id] = payload
    return out

def get_excluded_collaborators_ids(data, artist_ids, dump_ids):
    collections = data['collections']
    merged = set()
    for artist_id, payload in collections.items():
        collaborators = payload['collaborators']
        merged = merged.union(set(collaborators).intersection(dump_ids))
    out = merged - set(artist_ids)
    return out

def filter_data_by_ids(data, artist_ids):
    collections = data['collections']
    out = {
        'collections': {a_id: collections[a_id] for a_id in artist_ids}
    }
    return out

def filter_data_if_intersection_mteq_2(data, artist_ids):
    collections = data['collections']
    out = {'collections':{}}
    for artist_id, payload in collections.items():
        collaborators = payload['collaborators']
        intersected = set(collaborators).intersection(set(artist_ids))
        if len(intersected) >= 2:
            out['collections'][artist_id] = payload
            artist_ids.append(artist_id)
    return out

def update_db(queues_ids, excluded_indie_collaborator_data):
    print('updating_db....')
    # transaction = db.transaction()
    queue_doc_ref = db.collection('staging-dashboard').document('queue')
    # dump_coll_ref = db.collection('staging-exceeds_dump')
    # artist_collection_ref = db.collection('staging-artist_data')
    dashboard_queue_ref = db.collection('staging-dashboard').document('total_in_queue')
    # dashboard_nodes_ref = db.collection('staging-dashboard').document('total_nodes')

    batch = db.batch()
    # for queues_id in queues_ids:
        # payload = excluded_indie_collaborator_data['collections'][queues_id]
        # doc_ref = artist_collection_ref.document(queues_id)
        # dump_ref = dump_coll_ref.document(queues_id)
        # batch.set(doc_ref, payload)
        # print(f'delete from dump: {excluded_indie_collaborator_data["collections"][queues_id]["name"]}')
        # batch.delete(dump_ref)

    # total_nodes = dashboard_nodes_ref.get().get('value')
    batch.update(queue_doc_ref, {'ids':queues_ids})
    batch.update(dashboard_queue_ref, {'value':len(queues_ids)})
    # batch.update(dashboard_nodes_ref, {'value': total_nodes + len(queues_ids)})
    batch.commit()

def run():
    # get data
    dump_ids = load_ids(DUMP_PATH)
    artist_data_ids = load_ids(ARTIST_DATA_PATH)
    artist_data = load_data(ARTIST_DATA_PATH)
    dump_data = load_data(DUMP_PATH)
    # filter to indinesia only
    indienesia_data = indienesia_only(artist_data)
    # get ids of excluded indinesia's collaborators (in dump)
    excluded_indie_collaborator_ids = get_excluded_collaborators_ids(indienesia_data, artist_data_ids, dump_ids)
    # query exclude_indie_collaborator_ids from dump
    excluded_indie_collaborator_data = filter_data_by_ids(dump_data, excluded_indie_collaborator_ids)
    # filter out again which is associated to at least 2 artist_ids
    filtered_excluded_indie_collaborator_data = filter_data_if_intersection_mteq_2(excluded_indie_collaborator_data, artist_data_ids)
    first_queue_ids = list(filtered_excluded_indie_collaborator_data['collections'].keys())
    # print(first_queue_ids)
    update_db(first_queue_ids, filtered_excluded_indie_collaborator_data)

    # ------------
    # add to queue
    # push data & delete docuemnt to artist_id (1 batched transaction)
    # start mining

    # [LOCAL] optional, while mining
    # create reference = collaborators of filtered_excluded_indie_collaborator_data which in artist_data
    # update reference's collaborators --> add artist id of related filtered_excluded_indie_collaborator_data

    # recompute degree for each artist_ids in indienesia
    # if len == 1:
    # check original collaborators outside

run()


#!/usr/bin/env python3
'''
|
|------------------------------------------------------------------------------
|
|  PARSE_ARGUMENTS.PY
|
|  UPDATED:    2023-10-04
|  AUTHOR:     kent campbell
|  CONTACT:    campbellkb@ornl.gov
|
|  DESCRIPTION
|
|     This module provides functions for processing command line arguments
|
|------------------------------------------------------------------------------
|
'''
# | Python 3 standard library |
import argparse

from os.path import isdir, isfile
from sys import stdout

# | Global Variable(s) |
__version__ = '3.3.2'

# |----------------------------------------------------------------------------
# | command line argument handling functions
# |----------------------------------------------------------------------------
def argument_handler():
    '''Primary argument handler.'''

    # Parser object.
    p = argparse.ArgumentParser(description = '----- Granulemeta v' + __version__ + ' -----')

    # Determine where to send output. Defaults to stdout.
    p.add_argument('-o', '--outfile', default = stdout, type = str,
                   help = 'the path to an output file (with ext)')

    # Specify a configuration file at the command line
    p.add_argument('-c', '--config', default = None, type = _file_and_dir_validator,
                   help = 'a configuration file')

    # Exclude all other types but: 'raster', 'vector', 'mdim', 'table', ...
    p.add_argument('-t', '--type', type = _type_validator,
                   help = "input data type: [raster, vector, mdim, table, audio, envi, archive]")

    # Only process a certain number of granules before stopping execution
    p.add_argument('-n', '--nfiles', default = -1, type = int,
                   help = 'number of granules to process before stopping execution (defaults to -1 for all files).')

    # Determine where to send output. Defaults to stdout.
    p.add_argument('--outtype', default = 'gmout', type = str,
                   help = 'type of output file to produce')

    # If the output file already exists, overwrite its contents instead of appending new content only.
    p.add_argument('--overwrite', default = False, action = 'store_true',
                   help = 'a flag for enabling the overwriting of an existing output file')

    # Opt to format output for Daymet Lower Latency granules.
    p.add_argument('--daymet', default = False, action = 'store_true',
                   help = 'a flag indicating format for daymet')

    # Opt to keep endtime as is instead of adjusting ones that are at midnight.
    p.add_argument('--no-adjust-midnight-endtime', default = False, action = 'store_true',
                   help = 'a flag for disabling the auto-adjustment of the endtime')

    # Opt to remove aux.xml files from the input directory.
    p.add_argument('--keep_aux_xml', default = False, action = 'store_true',
                   help = 'a flag to disable the automatic deletion of aux.xml files')

    # Opt to leave bounding box and time extents blank when not found.
    p.add_argument('--force_blanks', default = False, action = 'store_true',
                   help = 'a flag to leave bounding box and time extents blank when not found instead of exiting with an error message')

    p.add_argument('--no_stats', default = False, action = 'store_true',
                   help = 'a flag to prevent calculating statistics on granules during execution')

    p.add_argument('--no_checksum', default = False, action = 'store_true',
                   help = 'a flag to prevent calculating the md5 checksum on granules during execution')

    p.add_argument('--add_ext', default = False, action = 'store_true',
                   help = 'a flag to add \'.bin\' extensions to extensionless ENVI files')

    p.add_argument('--lat_name', default = None, type = str,
                   help = 'a string providing the name of the column (in the granule) from which the script extracts the latitude values.')

    p.add_argument('--lon_name', default = None, type = str,
                   help = 'a string providing the name of the column (in the granule) from which the script extracts the longitude values.')

    p.add_argument('--date_name', default = None, type = str,
                   help = 'a string providing the name of the column (in the granule) from which the script extracts the date values.')

    p.add_argument('--time_name', default = None, type = str,
                   help = 'a string providing the name of the column (in the granule) from which the script extracts the time values.')

    p.add_argument('--version', default = False, action = 'store_true',
                   help = 'a flag to allow for the version number of granulemeta to be printed')

    # Positional argument 1: Input directory or file.
    p.add_argument('path', type = _file_and_dir_validator, nargs = '+',
                   help = 'an input directory (OR file OR sequence of files OR glob pattern(s)) to traverse for metadata')

    # Return parsed arguments.
    return p.parse_args()

def _type_validator(input_type: str):
    '''An input 'type' validator.'''

    # Valid inputs:
    v = ['raster', 'vector', 'mdim', 'table', 'audio', 'envi', 'archive']

    # Bad input message:
    e = "Input data type ('-t', '--type') must be from: {}.".format(v)

    # If the input data type is not valid, raise ValueError. Else return it.
    if input_type not in v:
        raise ValueError(e)
    else:
        return input_type

def _file_and_dir_validator(input_path: str):
    '''A file or directory path validator.'''
    if isinstance(input_path, str):
        if not any([isfile(input_path), isdir(input_path)]):
            print('No file(s) found corresponding to supplied PATH argument')
            exit(1)
        else:
            return input_path
    elif isinstance(input_path, list):
        for x in input_path:
            if not any([isfile(x), isdir(x)]):
                print('No file(s) found corresponding to supplied PATH argument')
                exit(1)
        return(input_path)


"""
A palindromic number reads the same both ways.
The largest palindrome made from the product of two 2-digit numbers is 9009 = 91 x 99.

Find the largest palindrome made from the product of two 3-digit numbers.
"""

def is_palindrome(number):
    if (len(str(number)) %2 == 0):
        digits_to_compare = len(str(number)) / 2
        for i in range(int(digits_to_compare)):
            if (str(number)[i] != str(number)[-(i+1)]):
                return False
        return True
    else:
        digits_to_compare = (len(str(number)) - 1) / 2
        for i in range(int(digits_to_compare)):
            if (str(number)[i] != str(number)[-(i+1)]):
                return False
        return True

def largest_palindrome():
    palindromes = []
    for i in range(999,99,-1):
        for j in range(999,99,-1):
            if is_palindrome(i*j):
                palindromes.append(i*j)
    return max(palindromes)

if __name__ == "__main__":
    print(largest_palindrome())


"""
Illustrates how to work with the kd_tree module.

"""

import kdtree

# Create an empty tree by specifying the number of
# dimensions its points will have
emptyTree = kdtree.create(dimensions=3)

# A kd-tree can contain different kinds of points, for example tuples
point1 = (2, 3, 4)

# Lists can also be used as points
point2 = [4, 5, 6]

# Other objects that support indexing can be used, too
import collections
Point = collections.namedtuple('Point', 'x y z')
point3 = Point(5, 3, 2)

# A tree is created from a list of points
tree = kdtree.create([point1, point2, point3])

# Each (sub)tree is represented by its root node
print(tree)

# Adds a tuple to the tree
tree.add( (5, 4, 3) )

# Removes the previously added point and returns the new root
tree = tree.remove( (5, 4, 3) )

# Retrieving the Tree in inorder
print(list(tree.inorder()))

# Retrieving the Tree in level order
print(list(kdtree.level_order(tree)))

# Find the nearest node to the location (1, 2, 3)
tree.search_nn( (1, 2, 3) )

# Add a point to make the tree more interesting
tree.add( (10, 2, 1) )

# Visualize the Tree
kdtree.visualize(tree)

# Take the right subtree of the root
subtree = tree.right

# and detatch it
tree.right = None
kdtree.visualize(tree)
kdtree.visualize(subtree)

# and re-attach it
tree.right = subtree
kdtree.visualize(tree)

# Add a node to make the tree unbalanced
print(tree.is_balanced)
tree.add( (6, 1, 5) )
tree.is_balanced
kdtree.visualize(tree)

# rebalance the tree
tree = tree.rebalance()
print(tree.is_balanced)
kdtree.visualize(tree)


from datetime import datetime, timedelta
from dataclasses import dataclass

@dataclass
class Item:
    x:int
    y:int
    move:int=0

class N(Item):
    def moveup(self):
        self.y  = self.y + 1
        self.move += 1
        return N(self.x, self.y)

class E(Item):
    def moveright(self):
        self.x += 1
        self.move +=1
        return E(self.x, self.y)

#global variables
Ngroup, Egroup, Infi = [], [], []
nItems = []
eItems = []
coordinateArray = []

def inputvalue():
    global Ngroup, Egroup
    print('Please input a integer for N between 0 and 50\n')
    total = input() # 6
    print(f'Please input {total} coordinates in the format "N 3 2,E 5 10" \n')
    # E 3 5, N 5 3, E 4 6, E 10 4, N 11 2, N 8 1
    coordinates = input()
    items = coordinates.strip().split(',')
    itemsarray =[i.strip().replace(' ','_') for i in items]
    print(f'items are {itemsarray}')
    # items are ['E_3_5', 'N_5_3', 'E_4_6', 'E_10_4', 'N_11_2', 'N_8_1']
    for i in itemsarray:
        c = i.split('_')
        if i.startswith('N'):
            Ngroup.append(c)
        elif i.startswith('E'):
            Egroup.append(c)
        else:
            print(i)
            print(f'{i} is not allowed, do the input again')
            inputvalue()
    print(f'{Ngroup} for Ngroup')
    print(f'{Egroup} for Egroup')
    # [['N', '5', '3'], ['N', '11', '2'], ['N', '8', '1']] for Ngroup
    # [['E', '3', '5'], ['E', '4', '6'], ['E', '10', '4']] for Egroup

    # Step 2
    doLogics()

def doLogics():
    print('Start comparing...')
    global Ngroup, Egroup, nItems, eItems
    if Ngroup == [] and Egroup != []:
        EmptyTheArrayAndReturn(Egroup)
    if Ngroup != [] and Egroup == []:
        EmptyTheArrayAndReturn(Ngroup)

    if Ngroup != [] and Egroup != []:
        consider3senarios()

    if Ngroup != [] and Egroup != []:
        doDetailCalculation(Ngroup,Egroup)

    print(f'Infinite are {Infi}')

def doDetailCalculation(Ngroup, Egroup):
    # turn all array items into N or E instances
    for n in Ngroup:
        nitem= N(x=int(n[1]), y=int(n[2]))
        nItems.append(nitem)
        coordinateArray.append((nitem.x, nitem.y))
    for e in Egroup:
        eitem = E(x=int(e[1]), y=int(e[2]))
        eItems.append(eitem)
        coordinateArray.append((eitem.x, eitem.y))

    print(f'CoordinateArray has {coordinateArray},\n nitems has {nItems},\n eitems has {eItems}')
    #CoordinateArray has [(5, 3), (11, 2), (8, 1), (3, 5), (4, 6), (10, 4)],
    #nitems has [N(x=5, y=3, move=0), N(x=11, y=2, move=0), N(x=8, y=1, move=0)],
    #eitems has [E(x=3, y=5, move=0), E(x=4, y=6, move=0), E(x=10, y=4, move=0)]

    # when all leftover Ns or Es will extend infinitely... use Timer to stop the program
    timer = datetime.now()
    while nItems != [] and eItems != []:
        # all N or E instances move one step at a time
        oneMove()

        start = datetime.now()
        delta = (start - timer)
        if (delta.total_seconds()) > 1:
            print(f'Take too long, looks like {eItems} and {nItems} are all infinite')
            break
    # coming out of the 'while' loop ...
    if nItems == [] and eItems != []:    # Infi item format is [['N', '5', '3'], ['N', '11', '2'], ['N', '8', '1']]
        for e in eItems:                  # eItems format is [E(x=3, y=5, move=0), E(x=4, y=6, move=0), E(x=10, y=4, move=0)]
            original = ['E',f'{e.x-e.move}',f'{e.y}']
            Infi.append(original)

    if eItems == [] and nItems != []:
        for n in nItems:
            original = ['N',f'{n.x}',f'{n.y-n.move}']
            Infi.append(original)

def consider3senarios():
    global Ngroup,Egroup
    '''One'''
    # find the largest Y in Egroup --> remove items in Ngroup that is above it:
    Ylargest = max([int(e[2]) for e in Egroup])
    print(f'Ylargest in Egroup is {Ylargest}')
    for n in Ngroup:
        # print(f'checking {n} in Ngroup to remove the top items')
        if int(n[2]) > Ylargest:
            Infi.append(n)
            # Ngroup.remove(n) # can't do this when iterate through this list
    # remove duplicate  items from Ngroup
    Ngroup = [elm for elm in Ngroup if elm not in Infi]

    '''Two'''
    # find the smallest X in Egroup --> remove items in Ngroup that is on its left side:
    if Ngroup != []:
        Xsmallest = min([int(e[1]) for e in Egroup])
        print(f'Xsmallest in Egroup is {Xsmallest}')
        for n in Ngroup:
            # print(f'checking {n} in Ngroup to remove the most left items')
            if int(n[1]) < Xsmallest:
                Infi.append(n)
        # remove the items from Ngroup if they are already moved to Infi array
        Ngroup = [elm for elm in Ngroup if elm not in Infi]

        if Ngroup == []:
            EmptyTheArrayAndReturn(Egroup)
        else:
            '''Three'''
            Xlargest = max([int(n[1]) for n in Ngroup])
            print(f'Xlargest in Ngroup is {Xlargest}')
            for e in Egroup:
                # print(f'checking {n} in Egroup to remove the item with larger x')
                if int(e[1]) > Xlargest:
                    Infi.append(e)
            # remove items from Egroup if they are in Infi array already
            Egroup = [elm for elm in Egroup if elm not in Infi]

    else:
        EmptyTheArrayAndReturn(Egroup)
    Ngroup = [elm for elm in Ngroup if elm not in Infi]
    Egroup = [elm for elm in Egroup if elm not in Infi]
    print(f'After 3 Senarios --> Infi has {Infi}, Ngroup has {Ngroup}, Egroup has {Egroup}')

def oneMove():
    global nItems,eItems
    print(f'moving one step now....')
    ''' N items change y->y+1, movement add 1;
            add (x, y+1) into zone[] if the tuple not existed yet
            else Stop, count movement, add n to infi[]
            E itmes change x-> x+1, movement add 1;
            add (x+1, y) into zone[] if the tuple not existed yet
            else stop, count movement, add n to infi[]
            update Ngroup and Egroup by comparing with infi[]
    '''
    # global a, b
    tempStore = []
    removeN=[]
    removeE = []
    print(f'In onemove() nItems is {nItems}')
    # In onemove() nItems is [N(x=5, y=3, move=0), N(x=11, y=2, move=0), N(x=8, y=1, move=0)]
    print(f'In onemove() eItems is {eItems}')
    # In onemove() eItems is [E(x=3, y=5, move=0), E(x=4, y=6, move=0), E(x=10, y=4, move=0)]
    for n in nItems:#N(x=5, y=3, move=0)
        a = n.moveup() #a is N(x=8, y=62, move=0)
        # After oneMove UP,n in nItems is N(x=5, y=4, move=1)
        # After oneMove UP,n in nItems is N(x=11, y=3, move=1)
        # After oneMove UP,n in nItems is N(x=8, y=2, move=1)
        aCor =(a.x, a.y)
        if aCor not in coordinateArray: #CoordinateArray has [(5, 3), (11, 2), (8, 1), (3, 5), (4, 6), (10, 4)]
            # print(f'line 186, add new coordinates to "coordinateArray"...')
            coordinateArray.append(aCor)
            print(f'CoordianteArray in 187 line has {coordinateArray}')
            tempStore.append(aCor)
            print(f'add new aCor to tempStore in 189 has {tempStore} items')
            # add new aCor to tempStore in 189 has [(5, 4), (11, 3), (8, 2)] items -> this is after all items are looped through
        else:
            # to remove collided items
            print(f'If aCor is already inside the coordiateArray.... line 192')
            original = f'N{n.x}{n.y-n.move}'
            print(f'original in 192 is {original}')
            print(f'{original} has {n.move} grass')
            # for remove it from nItems later
            removeN.append(n)
            print(f'removeN in 200 has {removeN}')
    for e in eItems:#E(x=3, y=5, move=0)
        b = e.moveright()
        bCor = (b.x, b.y)
        if bCor not in coordinateArray:
            coordinateArray.append(bCor)
        else:
            # if bCor is already stored in coordinateArray, check if it is just stored within this onemove()
            if bCor not in tempStore: # SO the bCor tuple has been stored in the previous oneMove()...
                # remove this b from eItem
                original = f'E{e.x - e.move}{b.y}'
                print(f'original in 210 is {original}')
                print(f'{original} has {e.move} grass')
                # for remove it from eItems later
                removeE.append(e)
                print(f'removeE in 215 has {removeE}')
            else: # so the bCor is just being stored in this session of oneMove()
                print(f'OOPS! E instance joined N instance in this {bCor} spot.')
    eItems = [i for i in eItems if i not in removeE] # eItems and nItems has E and N instances that are kept being updated after the oneMove()
    print(f'eItems has {eItems} LEFT')
    nItems = [i for i in nItems if i not in removeN]
    print(f'nItems has {nItems} LEFT')
    #eItems has [E(x=6, y=5, move=3), E(x=7, y=6, move=3), E(x=13, y=4, move=3)] LEFT
    #nItems has [N(x=5, y=6, move=3), N(x=11, y=5, move=3), N(x=8, y=4, move=3)] LEFT

# use this format for 2nd input:
# E 3 5, N 5 3, E 4 6, E 10 4, N 11 2, N 8 1

def EmptyTheArrayAndReturn(arr):
    Infi.extend(arr)
    print(f'EmptyTheArray from {arr} and returned -- Infi array has {Infi}')
    return

if __name__ =="__main__":
    inputvalue()


#!/usr/bin/env python3

"""
file contains function index_range
"""

def index_range(page: int, page_size: int) -> tuple:
    """
    Return a tuple of start index and end index for pagination.

    Args:
        page (int): The current page number(1-indexed).
        page_size (int): The number of items per page.

    Returns:
        tuple: A tuple containing the start index and end index.
    """
    if page <= 1:
        start_index = 0
    else:
        start_index = (page - 1) * page_size

    end_index = start_index + page_size

    return start_index, end_index


import ForecastHandler, ExpenseForecast
import BudgetSet, AccountSet, MemoRuleSet
import MilestoneSet, MemoMilestone, AccountMilestone

import CompositeMilestone

import logging
import datetime

log_format = '%(asctime)s - %(levelname)-8s - %(message)s'
l_formatter = logging.Formatter(log_format)

l_stream = logging.StreamHandler()
l_stream.setFormatter(l_formatter)
l_stream.setLevel(logging.INFO)

l_file = logging.FileHandler('scratch__'+datetime.datetime.now().strftime('%Y%m%d_%H%M%S')+'.log')
l_file.setFormatter(l_formatter)
l_file.setLevel(logging.INFO)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.propagate = False
logger.handlers.clear()
logger.addHandler(l_stream)
logger.addHandler(l_file)

if __name__ == '__main__':

    # account_set = AccountSet.AccountSet([])
    #
    # account_set.addAccount(name='Checking',
    #                        balance=1000,
    #                        min_balance=0,
    #                        max_balance=float('Inf'),
    #                        account_type="checking")
    #
    # budget_set = BudgetSet.BudgetSet([])
    # budget_set.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'Core', deferrable=False, partial_payment_allowed=False)
    #
    # account_milestone__list = []
    # memo_milestone__list = []
    # composite_milestone__list = []
    # milestone_names_for_composite_milestone = ['Account Milestone 1','Memo Milestone 1']
    #
    # account_milestone__list.append(AccountMilestone.AccountMilestone('Account Milestone 1','Checking',0,10000))
    # memo_milestone__list.append(MemoMilestone.MemoMilestone('Memo Milestone 1', '.*'))
    #
    # composite_milestone__list.append( CompositeMilestone.CompositeMilestone('composite milestone name 1',account_milestone__list,memo_milestone__list,milestone_names_for_composite_milestone) )
    #
    #
    #
    # MS = MilestoneSet.MilestoneSet(account_set,budget_set,account_milestone__list,memo_milestone__list,composite_milestone__list)

    F = ForecastHandler.ForecastHandler()

    F.initialize_from_excel_file('expense_forecast__milestone_testing__input.ods')

    F.run_forecasts()

    #print(F.read_results_from_disk())

    # E1 = ExpenseForecast.initialize_from_json_file(path_to_json='Forecast__2023_04_07__14_06_02__031534.json')
    # E1.appendSummaryLines()
    #
    # E2 = ExpenseForecast.initialize_from_json_file(path_to_json='Forecast__2023_04_07__13_27_36__020658.json')
    # E2.appendSummaryLines()

    #print(E1.evaulateMemoMilestone('EMT class'))

    #F.generateCompareTwoForecastsHTMLReport(E1,E2)

    # core_budget_set = [['1']]
    #
    # CoreBudgetSet = BudgetSet.BudgetSet([])
    # CoreBudgetSet.addBudgetItem('20000101','20000101',1,'once','1','Core',deferrable=False,partial_payment_allowed=False)
    #
    # BudgetSetA2 = BudgetSet.BudgetSet([])
    # BudgetSetA2.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'A2', deferrable=False, partial_payment_allowed=False)
    #
    # BudgetSetB2 = BudgetSet.BudgetSet([])
    # BudgetSetB2.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'B2', deferrable=False, partial_payment_allowed=False)
    #
    # BudgetSetC3 = BudgetSet.BudgetSet([])
    # BudgetSetC3.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'C3', deferrable=False, partial_payment_allowed=False)
    #
    # BudgetSetD3 = BudgetSet.BudgetSet([])
    # BudgetSetD3.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'D3', deferrable=False, partial_payment_allowed=False)
    #
    # BudgetSetE3 = BudgetSet.BudgetSet([])
    # BudgetSetE3.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'E3', deferrable=False, partial_payment_allowed=False)
    #
    # BudgetSetF4 = BudgetSet.BudgetSet([])
    # BudgetSetF4.addBudgetItem('20000101', '20000101', 1, 'once', '1', 'F4', deferrable=False, partial_payment_allowed=False)
    #
    # list_of_lists_of_budget_sets = [
    #     [ BudgetSetA2, BudgetSetB2 ],
    #     [ BudgetSetC3, BudgetSetD3, BudgetSetE3 ],
    #     [ BudgetSetF4 ]
    # ]
    #
    # F = ForecastHandler.ForecastHandler()
    #
    # account_set = AccountSet.AccountSet([])
    # memo_rule_set = MemoRuleSet.MemoRuleSet([])
    #
    # account_set.addAccount(name='Checking',
    #                        balance=1000,
    #                        min_balance=0,
    #                        max_balance=float('Inf'),
    #                        account_type="checking")
    #
    # memo_rule_set.addMemoRule(memo_regex='.*', account_from='Checking', account_to=None, transaction_priority=1)
    #
    # F.calculateMultipleChooseOne(account_set, CoreBudgetSet , memo_rule_set, '20000101', '20000103', list_of_lists_of_budget_sets)


# --- Imports --- #

import pytmx
import pygame
import sprites, local

# --- TiledMap Class --- #

class TiledMap(sprites.Sprite):
    def __init__(self, day=1, x=0):

        if day % 1 == 0:
            self.type = 0
        else:
            if int(day) % 5 == 0:
                self.type = 2
            else:
                self.type = 1

    @property
    def type(self):
    	return self.__type

    @type.setter
    def type(self, type):

        self.__type = type
        if self.__type == 1 or self.__type == 2:
            type = 1
        else:
            type = 0

        self.__tmxdata = pytmx.load_pygame(f"{local.IMAGE_PATH}/{type}.tmx", pixelalpha=True)
        self.width = self.__tmxdata.width * self.__tmxdata.tilewidth
        self.height = self.__tmxdata.height * self.__tmxdata.tileheight

        if hasattr(self, "rect"):
            x = self.rect.x
            self.update((self.width, self.height))
            self.rect.x = x
        else:
            super().__init__((self.width, self.height))
        self.render()

    def render(self):

        self.image.fill((0,)*4)
        for layer in self.__tmxdata.visible_layers:
            if isinstance(layer, pytmx.TiledTileLayer):
                for x, y, gid, in layer:
                    tile = self.__tmxdata.get_tile_image_by_gid(gid)
                    if tile:
                        self.image.blit(tile, (x * self.__tmxdata.tilewidth, y * self.__tmxdata.tileheight))

# --- Camera Class --- #

class Camera:
    def __init__(self, size):
        self.rect = pygame.rect.Rect((0,)*2, size)
        self.width = size[0]
        self.height = size[1]

    def apply(self, entity):

        if isinstance(entity, pygame.rect.Rect):
            return rect.move(self.rect.topleft)
        else:
            return entity.rect.move(self.rect.topleft)

    def update(self, target):
        # limit scrolling to map size

        x = -target.rect.x + local.DISPLAYW // 2
        # y = -target.rect.y + local.DISPLAYH // 2

        # x = min(0,x)
        # x = min(0, x)
        # x = max(-(self.weight - local.DISPLAYW), min(0, x))  # left/right
        # y = max(-(self.height - local.DISPLAYH), min(0, y))  # top/bottom
        self.rect = pygame.rect.Rect(x, 0, self.width, local.DISPLAYH)#self.height)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Implementation of fingerprints as presented as HI-DS 2015.
Fragments:
    - paths of size 3
Descriptors:
    - field descriptor (3-path) hybridization
Fingerprint:
    - count vector, no hashing
"""

import rdkit
from rdkit import Chem

__author__ = "Petr Škoda"
__license__ = "X11"
__email__ = "skoda@ksi.mff.cuni.cz"

METADATA = {
    "id": "method_rdkit_p3_f3_hybridization_tanimoto_like",
    "representation": "p3_f3_hybridization",
    "similarity": "tanimoto_like"
}

# region Representation calculation

def _pattern_fragment_generator(molecule, pattern):
    """

    :param molecule:
    :param pattern:
    :return: A generator of fragment for given molecule.
    """
    matches = molecule.GetSubstructMatches(Chem.MolFromSmarts(pattern))
    for atoms in matches:
        yield atoms

def _calculate_atom_code(atom_idx, field_map):
    """Create labeling for given atom from the given fields.

    :param atom_idx:
    :param field_map:
    :return:
    """
    # There may not be a value for every atom, in such case we will use "_".
    # This might be as we use bigger fragment to build field then to
    # construct fingerprint.
    if atom_idx in field_map:
        return field_map[atom_idx]
    else:
        return None

def _calculate_fragment_code(atoms, field_map):
    """Compute fragment code.

    :param atoms:
    :param field_map:
    :return: Score for given fragment as a vector of values.
    """
    codes = [""] * len(atoms)
    for i in range(0, len(atoms)):
        codes[i] = _calculate_atom_code(atoms[i], field_map)
    # canonization of the code vector
    beg = 0
    end = len(codes) - 1
    while beg < end:
        if codes[beg] > codes[end]:
            codes.reverse()
            break
        elif codes[beg] == codes[end]:
            beg += 1
            end -= 1
        else:
            break
    return codes

def _calculate_field_hybrid_descriptor(molecule, fragments):
    """Create field hybridization descriptor for each atom.

    :param molecule:
    :param fragments:
    :return: Dictionary <atom, label value>
    """
    value_map = {}
    counter_map = {}
    for atoms in fragments:
        # Compute average field value.
        average_value = 0.0
        for atom in atoms:
            average_value += molecule.GetAtomWithIdx(atom).GetHybridization()
        average_value /= len(atoms)
        # Save value to atoms.
        for atom in atoms:
            if atom not in value_map:
                value_map[atom] = 0
                counter_map[atom] = 0
            value_map[atom] += average_value
            counter_map[atom] += 1
    # Modify values as averages.
    for key in value_map.keys():
        value_map[key] /= float(counter_map[key])
    return value_map

def compute_representation(molecule):
    """Calculate and return representation of given molecule.
    The output is dictionary of objects, where each object has a value (id)
    and counter, which represent number of fragment instances.

    :param molecule:
    :return: Dictionary <label, {value, counter}>
    """
    fragments_field = list(_pattern_fragment_generator(molecule, "*~*~*"))
    field_map = _calculate_field_hybrid_descriptor(molecule, fragments_field)
    fragments = fragments_field
    representation = {}
    total_counter = 0
    for atoms in fragments:
        fragment = _calculate_fragment_code(atoms, field_map)
        fragment_as_str = str(fragment)
        if fragment_as_str not in representation:
            representation[fragment_as_str] = {
                "value": fragment,
                "instances": [],
                "counter": 0
            }
        representation[fragment_as_str]["counter"] += 1
        # Contains information about atoms, that form the fragment.
        representation[fragment_as_str]["instances"].append({
            "atoms": atoms
        })
        total_counter += 1
    return representation

def _fragment_score_label_similarity(left_fragment, right_fragment):
    """Given two fragments return their similarity.

    :param left_fragment:
    :param right_fragment:
    :return:
    """
    # Topology must be the same.
    if not left_fragment["value"] == right_fragment["value"]:
        return 0
    similarity = 0
    for index in range(1, len(left_fragment)):
        try:
            similarity += left_fragment[index] == right_fragment[index]
        except Exception as ex:
            raise ex
    return similarity

def similarity_tannimoto(left, right):
    """

    :param left: Dictionary <key, value>
    :param right: Dictionary <key, value>
    :return:
    """
    in_common = 0
    total = 0
    for left_fragment_key in left:
        for right_fragment_key in right:
            left_fragment = left[left_fragment_key]
            right_fragment = right[right_fragment_key]
            if not left_fragment["value"] == right_fragment["value"]:
                continue
            # Add shared number (ie. min) to in_common - intersenction.
            in_common += min(left_fragment["counter"],
                             right_fragment["counter"])
            # Add to total counter - union.
            total += left_fragment["counter"] + right_fragment["counter"]
    if not total == 0:
        # They have no fragments, this might happen if the molecules
        # are too small..
        return in_common / (float)(total)
    else:
        return 0

# endregion

# region Interface method implementation

def create_model(ligands, decoys):
    """Create and return query model.
    """
    return [compute_representation(molecule) for molecule in ligands]

def compute_score(model, query):
    """Compute and return activity likeness for given query molecule.
    """
    fingerprint = compute_representation(query)
    return max([similarity_tannimoto(fingerprint, ligand)
                for ligand in model])

# endregion


#!/usr/bin/env python3

from pprint import pprint
import sys
from swacc import operations, comunity, MAC

def main():
  if (len(sys.argv) > 3):
    devtp = sys.argv[1]
    oper = sys.argv[2]
    location = sys.argv[3]

    if ((devtp != 'sw') and (devtp != 'iot') and (devtp != 'pc')):
        print(' Device type is not valid ')
        sys.exit(2)
    if ((oper != 'reboot') and (oper != 'ping') and (oper != 'search') and (oper != 'ssh') and (oper != 'findport')):
        print(' Unknown operation ')
        sys.exit(2)
  else:
    print('Program needs a device category and location as argument !!!')
    sys.exit(2)

  if (devtp == 'sw'):
    ymlfile = 'sw.yml'
  elif (devtp == 'iot'):
    ymlfile = 'iot.yml'
  else:
    ymlfile = 'pc.yml'

  if (oper == 'ping'):
    operations.alive(ymlfile=ymlfile, location=location)
  elif (oper == 'reboot'):
    operations.rebootpi(ymlfile=ymlfile, location=location)
  elif (oper == 'ssh'):
    operations.ssh(ymlfile=ymlfile, location=location)
  elif (oper == 'findport'):
    # ex. macstr = "778e2c441d3a"
    macstr = sys.argv[4]
    if (len(sys.argv) > 5):
      macstr = sys.argv[5]
      comunity = sys.argv[4]

    splt_str = []
    if (len(macstr) == 12):
      for idx in range(0, len(macstr), 2):
        splt_str.append(macstr[idx: idx + 2])
      MAC = '.'.join([str(int(a,16)) for a in splt_str])
    else:
      print(' MAC address is not valid ')
      sys.exit(2)

    operations.findport(ymlfile=ymlfile, location=location, MAC=MAC, comunity=comunity)
  elif (oper == 'search'):
    for yt in swacc.filters.gen_dict_extract(ymlparsed['all'], location):
      pprint(yt)
  else:
    print('Command not found !!!')

if __name__ == "__main__":
    main()


import numpy as np

def graphConv(filename):
    size = 0
    with open(filename, 'r') as file:
        for line in file:
            contents = line.split()
            if contents[0] == 'p':
                size = int(contents[2])
                break
    adjmat = [[0 for x in range(size)] for x in range(size)]
    with open(filename, 'r') as file:
        for line in file:
            edge = line.split()
            if edge[0] == 'e':
                e1 = int(edge[1]) - 1
                e2 = int(edge[2]) - 1
                adjmat[e1][e2] = 1
                adjmat[e2][e1] = 1
    return adjmat

def complement(filename):
    G = graphConv(filename)
    for i in range(len(G[0])):
        for j in range(len(G[0])):
            if i != j:
                if G[i][j] == 0:
                    G[i][j] = 1
                else:
                    G[i][j] = 0
    return G

def rank(G):
    return np.linalg.matrix_rank(np.matrix(G))

#print(rank(complement("c125.txt")))
#print(rank(complement("c250.txt")))
#print(rank(complement("c500.txt")))