import numpy as np
import os
from flask import Flask, request, jsonify, render_template, url_for
import cv2
import numpy as np
import base64
import io
from PIL import Image

from werkzeug.utils import secure_filename

app = Flask(__name__)
UPLOAD_FOLDER = 'C:/Users/Nageshreddy/Desktop/imagescount/'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/index2')
def home2():
    return render_template('index2.html')

@app.route('/predict1',methods=['POST'])
def predict1():

    print(request)
    f = request.files['file']
    f.save(os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(f.filename)))
    image = cv2.imread(UPLOAD_FOLDER+f.filename)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    blur = cv2.GaussianBlur(gray, (11, 11), 0)
    canny = cv2.Canny(blur, 30, 100, 3)
    dilated = cv2.dilate(canny, (1, 1), iterations=0)

    (cnt, hierarchy) = cv2.findContours(dilated.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    cv2.drawContours(rgb, cnt, -1, (0, 255, 0), 2)
    im = Image.open(UPLOAD_FOLDER+f.filename)
    print("items in image ", len(cnt))
    data1 = io.BytesIO()
    im.save(data1, 'JPEG')
    prediction = len(cnt)
    encoded_img_data = base64.b64encode(data1.getvalue())

    return render_template('index.html', prediction_text1='Items in image {}'.format(prediction),
                           image_name1= f.filename, uploaded_image1=encoded_img_data.decode('utf-8'))

@app.route('/predict2',methods=['POST'])
def predict2():

    print(request)
    f = request.files['file']

    f.save(os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(f.filename)))
    img = cv2.imread(UPLOAD_FOLDER+f.filename)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    _, thresh = cv2.threshold(img, 225, 255, cv2.THRESH_BINARY_INV)
    kernal = np.ones((2, 2), np.uint8)
    dilation = cv2.dilate(thresh, kernal, iterations=2)
    contours, hierarchy = cv2.findContours(dilation, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    objects = len(contours)
    im = Image.open(UPLOAD_FOLDER+f.filename)
    print("items in image ", objects)
    data1 = io.BytesIO()
    im.save(data1, 'JPEG')

    encoded_img_data = base64.b64encode(data1.getvalue())

    return render_template('index2.html', prediction_text2='Items in image {}'.format(objects),
                           image_name2= f.filename, uploaded_image2=encoded_img_data.decode('utf-8'))

@app.route('/predict_api',methods=['POST'])
def predict_api():
    data = request.get_json(force=True)
    prediction = model.predict([np.array(list(data.values()))])

    output = prediction[0]
    return jsonify(output)

@app.route('/success', methods = ['POST'])
def success():
    if request.method == 'POST':
        f = request.files['file']
        f.save(f.filename)
        return render_template("success.html", name = f.filename)

if __name__ == "__main__":
    app.run(debug=True)


# 숫자 입력값
a = int(input())

for i in range(0, a):
    num = 0
    b = str(input())
    for j in range(0, len(b)):
        # 문자열을 각각 하나의 문자마다 체크해야하기 때문에 배열형태로 된 문자열 인덱싱 활용
        if b[j] == "(":
            # 스택 idea --> "("를 넣고 "("를 빼는{=")"를 넣는} 형식 --> LIFO(Last In First Out) 방식
            num += 1
        elif b[j] == ")":
            num -= 1
        if num < 0:
            print("NO")
            break
    # 올바른 괄호가 되려면 무조건 "("의 갯수와 ")"의 갯수가 일치해야함
    if num == 0:
        print("YES")
    # ")"가 더 많은 경우(즉, num < 0)는 이미 위에서 처리하기 때문에 "("가 더 많은 경우(즉, num > 0)만 처리
    elif num > 0:
        print("NO")

단계별/스택 사용하기/9012.py


#ЛАБА №7 Сортировка Шелла

from random import randint

#создаем случайную последовательность
x = int(input("Введите размер списка \n"))
nums = [randint(1, 100) for i in range(0, x)]
print(*nums)

step=len(nums)//2
while step>0:
    for i in range(step, len(nums)):
        j=i
        while j>=step and nums[j]<nums[j-step]:
            nums[j], nums[j - step]=nums[j - step], nums[j]
            j=j-step
    step=step//2
print(*nums)


import hydra
from omegaconf import DictConfig, OmegaConf
import logging

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet18, resnet50
from tqdm import tqdm
import matplotlib.pyplot as plt
import pickle
import time
from sklearn.metrics import confusion_matrix
from utils import Dataset, fix_seed, make_folder, get_rampup_weight, cal_OP_PC_mIoU, save_confusion_matrix
from losses import PiModelLoss, ProportionLoss, VATLoss
from PIL import Image
from Model.model_backward import Resnet18_k8 as Resnet18
from torch.autograd import Variable
from utils import *

log = logging.getLogger(__name__)

@ hydra.main(config_path='../config', config_name='ours_backward_k8')
def main(cfg: DictConfig) -> None:

    # file name
    cwd = hydra.utils.get_original_cwd()
    result_path = cwd + cfg.result_path
    result_path += 'wsi-proportion-loss/'
    make_folder(result_path)
    result_path += cfg.consistency
    result_path += '-minibatch_%s' % str(cfg.mini_batch)
    result_path += '-samp_%s' % str(cfg.num_sampled_instances)
    result_path += '-lr_%s' % str(cfg.lr)
    result_path += '-seed_%s' % str(cfg.seed)
    result_path += '/'
    make_folder(result_path)

    fh = logging.FileHandler(result_path+'exec.log')
    log.addHandler(fh)
    log.info(OmegaConf.to_yaml(cfg))
    log.info('cwd:%s' % cwd)

    dataset_path = cwd + cfg.dataset_dir

    train_index, train_wsi_name, train_path, train_label, train_proportion = \
        load_data(dataset_path, 'train')

    _, val_wsi_name, val_path, val_label, val_proportion = \
        load_data(dataset_path, 'val')

    _, test_wsi_name, test_path, test_label, test_proportion = \
        load_data(dataset_path, 'test')

    # define loader
    train_dataset = DatasetBagSampling(
        name=train_wsi_name,
        path=train_path,
        label=train_label,
        proportion=train_proportion,
        num_sampled_instances=cfg.num_sampled_instances,
        cwd=cwd)
    train_loader = torch.utils.data.DataLoader(
        train_dataset, sampler=ImbalancedDatasetSampler(train_wsi_name), batch_size=1,
        shuffle=False,  num_workers=cfg.num_workers)

    val_dataset = DatasetBag(
        name=val_wsi_name,
        path=val_path,
        label=val_label,
        proportion=val_proportion,
        cwd=cwd)
    val_loader = torch.utils.data.DataLoader(
        val_dataset, batch_size=1,
        shuffle=False,  num_workers=4)

    test_dataset = DatasetBag(
        name=test_wsi_name,
        path=test_path,
        label=test_label,
        proportion=test_proportion,
        cwd=cwd)
    test_loader = torch.utils.data.DataLoader(
        test_dataset, batch_size=1,
        shuffle=False,  num_workers=4)

    # define model
    fix_seed(cfg.seed)
    if cfg.model == 'resnet50':
        model = resnet50(weights='ResNet50_Weights.DEFAULT')
    elif cfg.model == 'resnet18':
        # model = resnet18(pretrained=True)
        model = Resnet18(3)
    else:
        log.info('No model!')
    # model.fc = nn.Linear(model.fc.in_features, cfg.num_classes)
    model = model.to(cfg.device)

    # define criterion and optimizer
    loss_function = ProportionLoss(metric=cfg.proportion_metric)
    if cfg.consistency == 'none':
        consistency_criterion = None
    elif cfg.consistency == 'vat':
        consistency_criterion = VATLoss()
    elif cfg.consistency == 'pi':
        consistency_criterion = PiModelLoss()
    else:
        raise NameError('Unknown consistency criterion')

    l1_function = ProportionLoss(metric=cfg.val_metric)
    optimizer = torch.optim.Adam(model.parameters(), lr=cfg.lr)

    fix_seed(cfg.seed)
    # train_acces, test_acces = [], []
    train_OPs, train_PCs, train_mIoUs = [], [], []
    test_OPs, test_PCs, test_mIoUs = [], [], []
    train_losses, val_losses, test_losses = [], [], []
    best_validation_loss = np.inf
    final_OP, final_PC, final_mIoU = 0, 0, 0
    for epoch in range(cfg.num_epochs):
        s_time = time.time()

        # train
        model.train()
        losses = []
        b_list = [0]
        mb_data, mb_proportion = [], []
        gt, pred = [], []
        for iteration, (data, label, proportion) in enumerate(tqdm(train_loader, leave=False)):
            data = data[0]
            gt.extend(label[0].numpy())
            b_list.append(b_list[-1]+data.size(0))
            mb_data.extend(data)
            mb_proportion.extend(proportion)

            if (iteration+1) % cfg.mini_batch == 0 or (iteration + 1) == len(train_loader):
                mb_data = torch.stack(mb_data)
                mb_proportion = torch.stack(mb_proportion)
                mb_data = mb_data.to(cfg.device)
                mb_proportion = mb_proportion.to(cfg.device)

                if cfg.consistency == "vat":
                    # VAT should be calculated before the forward for cross entropy
                    consistency_loss = consistency_criterion(model, mb_data)
                elif cfg.consistency == "pi":
                    consistency_loss, _ = consistency_criterion(model, mb_data)
                else:
                    consistency_loss = torch.tensor(0.)
                alpha = get_rampup_weight(0.05, iteration, -1)
                consistency_loss = alpha * consistency_loss

                ft = []
                ft.append(mb_data.cpu())
                init_flag = True
                for i in range(len(model.net)):
                    if init_flag:
                        for j in range(len(model.net) - 1):
                            model.net[j].requires_grad = False
                        model.net[-1].requires_grad = True
                        init_flag = False
                    ft[-1] = Variable(ft[-1], requires_grad=True)
                    ft.append(model(ft[-1].to(cfg.device), i).cpu())
                pred.extend(ft[-1].argmax(1).cpu().detach().numpy())
                confidence = F.softmax(ft[-1], dim=1)
                pred_prop = torch.zeros(mb_proportion.size(
                    0), cfg.num_classes).to(cfg.device)
                for n in range(mb_proportion.size(0)):
                    pred_prop[n] = torch.mean(
                        confidence[b_list[n]: b_list[n+1]], dim=0)

                prop_loss = loss_function(pred_prop, mb_proportion)
                loss = prop_loss + consistency_loss
                loss.backward()
                ft = ft[:-1]
                grad, ft = get_grad(ft)
                optimizer.step()
                model.net[-1].requires_grad= False

                b_list = [0]
                mb_data, mb_proportion = [], []
                losses.append(loss.item())

                for i in reversed(range(len(model.net)-1)):
                    optimizer.zero_grad()
                    for j in range(i):
                        model.net[0].requires_grad = False
                    model.net[i].requires_grad = True
                    output = model(ft[-1].to(cfg.device), i)
                    loss = torch.sum((grad.to(cfg.device) * output), dim=1).mean()
                    loss.backward()
                    assert grad.shape[1] == output.shape[1], 'difference size error'
                    grad, ft = get_grad(ft)
                    optimizer.step()
                    model.net[i].requires_grad = False
                    output = output.detach().cpu()

        train_loss = np.array(losses).mean()
        train_losses.append(train_loss)

        # train_acc = np.array(np.array(gt) == np.array(pred)).mean()
        train_cm = confusion_matrix(y_true=gt, y_pred=pred)
        train_OP, train_PC, train_mIoU = cal_OP_PC_mIoU(train_cm)
        train_OPs.append(train_OP)
        train_PCs.append(train_PC)
        train_mIoUs.append(train_mIoU)

        e_time = time.time()
        log.info('[Epoch: %d/%d (%ds)] train OP: %.4f, PC: %.4f, mIoU: %.4f' %
                 (epoch+1, cfg.num_epochs, e_time-s_time, train_OP, train_PC, train_mIoU))

        # validation
        s_time = time.time()
        val_l1, val_cm, val_OP, val_PC, val_mIoU = evaluation(
            model, val_loader, cfg)
        e_time = time.time()
        val_losses.append(val_l1)
        log.info('[Epoch: %d/%d (%ds)] val l1: %.4f, OP: %.4f, PC: %.4f, mIoU: %.4f' %
                 (epoch+1, cfg.num_epochs, e_time-s_time,
                  val_l1, val_OP, val_PC, val_mIoU))
        # test
        s_time = time.time()
        test_l1, test_cm, test_OP, test_PC, test_mIoU = evaluation(
            model, test_loader, cfg)
        test_OPs.append(test_OP)
        test_PCs.append(test_PC)
        test_mIoUs.append(test_mIoU)
        test_losses.append(test_l1)

        e_time = time.time()
        log.info('[Epoch: %d/%d (%ds)] test l1: %.4f, OP: %.4f, PC: %.4f, mIoU: %.4f' %
                 (epoch+1, cfg.num_epochs, e_time-s_time,
                  test_l1, test_OP, test_PC, test_mIoU))
        log.info(
            '-----------------------------------------------------------------------')

        if val_l1 < best_validation_loss:
            torch.save(model.state_dict(), result_path + 'best_model.pth')
            save_confusion_matrix(cm=train_cm, path=result_path+'cm_train.png',
                                  title='epoch: %d, OP: %.4f, PC: %.4f, mIoU: %.4f' % (epoch+1, train_OP, train_PC, train_mIoU))
            save_confusion_matrix(cm=val_cm, path=result_path+'cm_val.png',
                                  title='epoch: %d, l1: %.4f, OP: %.4f, PC: %.4f, mIoU: %.4f' % (epoch+1, val_l1, val_OP, val_PC, val_mIoU))
            save_confusion_matrix(cm=test_cm, path=result_path+'cm_test.png',
                                  title='epoch: %d, OP: %.4f, PC: %.4f, mIoU: %.4f' % (epoch+1, test_OP, test_PC, test_mIoU))

            best_validation_loss = val_l1
            final_OP = test_OP
            final_PC = test_PC
            final_mIoU = test_mIoU

        np.save(result_path+'train_OP', train_OPs)
        np.save(result_path+'train_PC', train_PCs)
        np.save(result_path+'train_mIoU', train_mIoUs)
        np.save(result_path+'test_OP', test_OPs)
        np.save(result_path+'test_PC', test_PCs)
        np.save(result_path+'test_mIoU', test_mIoUs)
        plt.plot(train_OPs, label='train_OP')
        plt.plot(train_PCs, label='train_PC')
        plt.plot(train_mIoUs, label='train_mIoU')
        plt.plot(test_OPs, label='test_OP')
        plt.plot(test_PCs, label='test_PC')
        plt.plot(test_mIoUs, label='test_mIoU')
        plt.legend()
        plt.ylim(0, 1)
        plt.xlabel('epoch')
        plt.ylabel('acc')
        plt.savefig(result_path+'curve_acc.png')
        plt.close()

        np.save(result_path+'train_loss', train_losses)
        np.save(result_path+'val_loss', val_losses)
        plt.plot(train_losses, label='train_loss')
        plt.plot(val_losses, label='val_loss')
        plt.plot(test_losses, label='test_loss')
        plt.legend()
        plt.xlabel('epoch')
        plt.ylabel('loss')
        plt.savefig(result_path+'curve_loss.png')
        plt.close()

    log.info(OmegaConf.to_yaml(cfg))
    log.info('OP: %.4f, PC: %.4f, mIoU: %.4f' %
             (final_OP, final_PC, final_mIoU))
    log.info('--------------------------------------------------')

if __name__ == '__main__':
    main()


import base_functions as adv
import cv2
import numpy as np
import re, sys, os, time, json
from collections import defaultdict

import PIL
from PIL import Image
import PIL.Image
from pytesseract import image_to_string
import pytesseract

from pdf2image import convert_from_path

# import the necessary packages
from lenet import LeNet
from sklearn.model_selection import train_test_split
from keras.datasets import mnist
from keras.optimizers import SGD
from keras.utils import np_utils
from keras import backend as K
import keras
import numpy as np
import argparse
import cv2

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--pdf-file", type=str, default=-1, help="(mandatory) provide pdf file to be parsed")
ap.add_argument("-s", "--save-model", type=int, default=-1, help="(optional) whether or not model should be saved to disk")
ap.add_argument("-l", "--load-model", type=int, default=-1,	help="(optional) whether or not pre-trained model should be loaded")
ap.add_argument("-w", "--weights", type=str, help="(optional) path to weights file")
args = vars(ap.parse_args())
#print(args)

pytesseract.pytesseract.tesseract_cmd = 'C:/Program Files (x86)/Tesseract-OCR/tesseract'
TESSDATA_PREFIX = 'C:/Program Files (x86)/Tesseract-OCR'

imSects = {
           'pg1' : {
			  'form_hdr_1'  : [0, 100, 0, 200],
              'form_hdr_2'  : [0, 240, 460, 2000],
              'form_hdr_3'  : [0, 230, 2020, 2400],
              'name'        : [760, 860, 0, 1500],
			  'country'     : [760, 860, 1500, 2400],
              'disr_ent'    : [860, 920, 0, 2400],
              'ch_3_sts_a'  : [930, 1100, 0, 2500],
			  'ch_3_sts_b'  : [1100, 1200, 0, 2500],
              'ch_4_sts'    : [1200, 2380, 0, 2500],
              'perm_addr'   : [2380, 2490, 0, 2500],
              'perm_city'   : [2480, 2580, 0, 1200],
              'perm_ctry'   : [2480, 2580, 1850, 2400],
              'mail_addr'   : [2580, 2660, 0, 2500],
			  'mail_city'   : [2670, 2750, 0, 1850],
              'mail_ctry'   : [2670, 2750, 1850, 2400],
              'us_tin'      : [2760, 2860, 0, 700],
			  'giin'        : [2760, 2860, 950, 1350],
			  'f_tin'       : [2760, 2860, 1350, 1780],
			  'ref_nbr'     : [2760, 2860, 1800, 2500]
	      },
		  'pg2' : {
			  'part2'       : [190, 760, 0, 2500],
              'part3'       : [760, 1350, 0, 2500],
              'part4'       : [1420, 2170, 0, 2500],
			  'part5'       : [2240, 2890, 0, 2500]
		  }
		 }

ch_3_sts = {
            '1'  : 'Corporation',
			'2'  : 'Disregarded entity',
            '3'  : 'Partnership',
			'4'  : 'Simpie trust',
            '5'  : 'Grantor trust',
            '6'  : 'Complex trust',
            '7'  : 'Estate',
            '8'	 : 'Government',
			'9'  : 'Central Bank of Issue',
            '10' : 'Tax-exempt organization',
            '11' : 'Private foundation',
			'12' : 'Yes',
            '13' : 'No'
		}
ch_4_sts = {
 '1' : 'Nonparticipating FFI (including a limited FFI or an FFI related to. Reporting IGA FFI other than a registered deemed-compliant FF or participating FFI).',
 '2' : 'Participating FFI.',
 '3' : 'Reporting Model 1 FFI.',
 '4' : 'Reporting Model 2 FFI.',
 '5' : 'Registered deemed-compliant FFI (other than a reporting Model FFI or sponsored FFI that has not obtained a GIIN).',
 '6' : 'Sponsored FFI that has not obtained a GIIN. Complete Part IV.',
 '7' : 'Certified deemed-compiiant nonregistering local bank. Compltet Part V.',
 '8' : 'Certified deemed-compliant FFS with only low-value accounts. Complete Part VI.',
 '9' : 'Certified deemed-compliant sponsored, closely held investmen: vehicle. Complete Part VII.',
 '10' : 'Certified deemed-compliant limited life debt investment entity. Complete Part VIII',
 '11' : 'Certitied deemed-campliant investment advisors and investment managers. Complete Part IX.',
 '12' : 'Owner-documented FFI. Complete Part X.',
 '13' : 'Restricted distributor. Complete Part XI.',
 '14' : 'Nonreporting IGA FFI (including an FF! treated as a registered deemed-compliant FFI under an applicable Model 2 IGA). Complete Part XIl.',
 '15' : 'Foreign government, government of a U.S. possession, or foreign central bank of issue. Complete Part XIII.',
 '16' : 'International organization. Complete Part XIV.',
 '17' : 'Exempt retirement plans. Complete Part XV.',
 '18' : 'Entity wholly owned by exempt beneficial owners. Complete Part XVI.',
 '19' : 'Territory financial institution. Complete Part XVII.',
 '20' : 'Nonfinancial group entity. Complete Part XVIII.',
 '21' : 'Excepted nonfinancial start-up company. Complete Part XIX.',
 '22' : 'Excepted nonfinancial entity in liquidation or bankruptcy. Complete Part XX.',
 '23' : '501(c) organization. Compiete Part XXI.',
 '24' : 'Nonprofit organization. Complete Part XXII.',
 '25' : 'Publicly traded NFFE or NFFE affiliate of a publicly traded corporation. Complete Part XXIlE.',
 '26' : 'Excepted territory NFFE. Complete Part XXIV.',
 '27' : 'Active NFFE. Complete Part XXVV.',
 '28' : 'Passive NFFE. Complete Part XXVI.',
 '29' : 'Excepted inter-affiliate FFI. Complete Part XXVII.',
 '30' : 'Direct reporting NFFE.',
 '31' : 'Sponsored direct reporting NFFE. Compiete Part XXVIII.'
}

# variables - folders and imgae files
w8_content = {}
for i in range(30):
    w8_content['part'+('0'+str(i+1) if i < 9 else str(i+1))] = {}

binOpt = {'1' : "Yes", '2' : "No"}
folder = os.path.join('static', 'files')
tmp_folder=os.path.join('static', 'tmp')

def get_pages(pdf_file):
  pages = convert_from_path(pdf_file, 300)
  head, tail = os.path.split(pdf_file)
  fileName = tail.split('.')[0]
  directory = os.path.join(folder, fileName)
  if not os.path.exists(directory):
     os.makedirs(directory)
  page_names = []
  for page in pages:
      imgfile = "{}-page_{}.jpg".format(fileName, pages.index(page))
      page.save(os.path.join(directory, imgfile), "JPEG")
      page_names.append(os.path.join(directory, imgfile))
  return page_names

def get_area(img, pg_num, sect_nm):
    return img[imSects[pg_num][sect_nm][0]:imSects[pg_num][sect_nm][1], imSects[pg_num][sect_nm][2]:imSects[pg_num][sect_nm][3]]

def checkbox(img, filename, txtWidth=0, count=1, meth="top-to-bottom", folder=tmp_folder):
    img = cv2.cvtColor(img,cv2.COLOR_GRAY2RGB)
    if len(img.shape) == 3:
       gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    else:
       gray = img

    ret,thresh = cv2.threshold(gray,127,255,cv2.THRESH_BINARY_INV)
    im2,ctrs, hier = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    #sort contours
    (sorted_ctrs, boundingBoxes) = adv.sort_contours(ctrs, method=meth)
    fresh = 255 - thresh
    chkbx_arr = []; obj = {}
    for c in sorted_ctrs:
        # approximate the contour
        peri = cv2.arcLength(c, True)
        approx = cv2.approxPolyDP(c, 0.15 * peri, True)
        # if  approximated contour has four points, then
        # it is assumed that it is a checkboxes
        x, y, w, h = cv2.boundingRect(c)
        if len(approx) == 4 and w > 35:
           roi = fresh[y:y+h, x:x+w]
           #adv.showim(roi_desc)
           sum = np.sum(cv2.resize(roi, (50, 50)) == 255)
           if txtWidth > 0:
              roi_desc = pytesseract.image_to_string(Image.fromarray(fresh[y:y+h, x+w:x+w+txtWidth]).convert("RGB"), lang='eng')
              obj[roi_desc] = sum
              cv2.rectangle(img,(x,y),( x + w + txtWidth, y + h ),(0,0,255),2)
              cv2.putText(img, str(count), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2)
           else:
              cv2.rectangle(img,(x,y),( x + w, y + h ),(0,0,255),2)
              cv2.putText(img, str(count), (x-w, y+h), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2)
              chkbx_arr.append(sum)
           #adv.showim(img)
           count += 1
    filename = filename if len(filename.split('.')) > 1 else filename+'.jpg'
    cv2.imwrite(os.path.join(folder, filename), img)
    return obj if txtWidth > 0 else chkbx_arr

def get_line_vals(arg, filename, folder=tmp_folder, count=1):
    if type(arg) is np.ndarray:
       img = arg.copy()
    else:
       img = cv2.imread(arg, 0)
    dimg = hlinesImg(img, iter=10, vdilate=1); #adv.showim(dimg);
    cv2.imwrite(os.path.join(folder, filename), dimg)
    img = cv2.cvtColor(img,cv2.COLOR_GRAY2RGB)
    #if len(dimg.shape) == 3:
    #   gray = cv2.cvtColor(dimg,cv2.COLOR_BGR2GRAY)
    #else:
    #   gray = dimg
    #ret,thresh = cv2.threshold(gray,127,255,cv2.THRESH_BINARY_INV)
    im2, ctrs, hier = cv2.findContours(dimg.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    #sort contours
    (sorted_ctrs, boundingBoxes) = adv.sort_contours(ctrs, method="top-to-bottom")
    lines_arr = []
    for c in sorted_ctrs:
        x, y, w, h = cv2.boundingRect(c)
        output = pytesseract.image_to_string(Image.fromarray(img[y+h//2-50:y+h//2, x:x+w]).convert("RGB"), lang='eng')
        cv2.rectangle(img,(x,y+h//2-50),( x + w, y + h//2 ),(0,0,255),2);#adv.showim(img)
        cv2.putText(img, str(count), (x+10, y+10), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2)
        count += 1
        lines_arr.append(output)
    cv2.imwrite(os.path.join(folder, filename), img)
    return lines_arr

def pg_scan(page, pg_name, idx):
    obj = checkbox(page, pg_name)
    for key in range(len(obj)):
        if obj[key] < 1300:
           w8_content[idx[key].split(':')[0]][idx[key].split(':')[1]] = 'Yes'

if __name__ == "__main__":
   start = time.time()
   pdf_file = args["pdf_file"]
   page_names = get_pages(pdf_file)
   page_1 = page_names[0]
   page_2 = page_names[1]
   page_3 = page_names[2]
   page_4 = page_names[3]
   page_5 = page_names[4]
   page_6 = page_names[5]
   page_7 = page_names[6]
   page_8 = page_names[7]
   # crop first page and set standard size of (width = 2500, height = 3000)
   form_c = cv2.resize(adv.crop_form(page_1), (2500, 3000))

   #----------------------------------
   '''     P  A  G  E   -   1      '''
   #----------------------------------

   # ----------------------------------------------------------------------
   # ------------ PAGE 1 - PART I  -  1, 2, 3  ----------------------------
   # ----------------------------------------------------------------------
   name = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'name')).convert("RGB"), lang='eng').splitlines()
   country = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'country')).convert("RGB"), lang='eng').splitlines()
   disr_ent = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'disr_ent')).convert("RGB"), lang='eng').splitlines()

   w8_content['part01']['01'] = name[-1] if len(name) > 1 else ''
   w8_content['part01']['02'] = country[-1] if len(country) > 1 else ''
   w8_content['part01']['03'] = disr_ent[-1] if len(disr_ent) > 1 else ''

   ## ----------------------------------------------------------------------
   ## ------------  PAGE 1 - PART I - 4 (Chapter 3 Status)  ----------------
   ## ----------------------------------------------------------------------

   #checkbox(img, filename, txtWidth=0, count=1, folder='D:\\DB_OCR\\tmp')
   w8_content['part01']['04a'] = []

   obj_a = checkbox(get_area(form_c, 'pg1', 'ch_3_sts_a'), 'ch-3-sts-a', txtWidth=400)
   for key in obj_a.keys():
       if obj_a[key] < 1300:
          w8_content['part01']['04a'].append(key)

   obj_b = checkbox(get_area(form_c, 'pg1', 'ch_3_sts_b'), 'ch-3-sts-b', meth="left-to-right")
   for i in range(len(obj_b)):
       if obj_b[i] < 1300: w8_content['part01']['04b'] = binOpt[str(i+1)]

   ## ----------------------------------------------------------------------
   ## ------------  PAGE 1 - PART I - 5 (Chapter 4 Status)  ----------------
   ## ----------------------------------------------------------------------
   img = get_area(form_c, 'pg1', 'ch_4_sts')
   w8_content['part01']['05'] = []

   h, w = img.shape
   obj_a = checkbox(img[:, 0:w//2], 'ch-4-sts-a')
   obj_b = checkbox(img[:, w//2:w], 'ch-4-sts-b', count=len(obj_a)+1)
   ch4_arr = [obj_a.index(x) for x in obj_a if x < 1300] + [obj_a.index(y) for y in obj_b if y < 1300]
   w8_content['part01']['05'] = [ch_4_sts[str(key)] for key in ch4_arr]

   ## ----------------------------------------------------------------------
   ## ------------  PAGE 1 - PART I - 6, 7, 8, 9a, 9b, 10   ----------------
   ## ----------------------------------------------------------------------
   perm_addr = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'perm_addr')).convert("RGB"), lang='eng').splitlines()
   perm_city = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'perm_city')).convert("RGB"), lang='eng').splitlines()
   perm_ctry = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'perm_ctry')).convert("RGB"), lang='eng').splitlines()
   mail_addr = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'mail_addr')).convert("RGB"), lang='eng').splitlines()
   mail_city = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'mail_city')).convert("RGB"), lang='eng').splitlines()
   mail_ctry = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'mail_ctry')).convert("RGB"), lang='eng').splitlines()
   us_tin    = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'us_tin')).convert("RGB"), lang='eng').splitlines()
   giin      = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'giin')).convert("RGB"), lang='eng').splitlines()
   f_tin     = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'f_tin')).convert("RGB"), lang='eng').splitlines()
   ref_nbr   = pytesseract.image_to_string(Image.fromarray(get_area(form_c, 'pg1', 'ref_nbr')).convert("RGB"), lang='eng').splitlines()

   w8_content['part01']['06']  = perm_addr[-1] if len(perm_addr) > 1 else ''
   w8_content['part01']['06a'] = perm_city[-1] if len(perm_city) > 1 else ''
   w8_content['part01']['06b'] = perm_ctry[-1] if len(perm_ctry) > 1 else ''
   w8_content['part01']['07']  = mail_addr[-1] if len(mail_addr) > 1 else ''
   w8_content['part01']['07a'] = mail_city[-1] if len(mail_city) > 1 else ''
   w8_content['part01']['07b'] = mail_ctry[-1] if len(mail_ctry) > 1 else ''
   w8_content['part01']['08']  = us_tin[-1]    if len(us_tin)    > 1 else ''
   w8_content['part01']['09a'] = giin[-1]      if len(giin)      > 1 else ''
   w8_content['part01']['09b'] = f_tin[-1]     if len(f_tin)     > 1 else ''
   w8_content['part01']['10'] = ref_nbr[-1]    if len(ref_nbr)   > 1 else ''

   # crop second page and set standard size of (width = 2500, height = 3000)
   form_c = cv2.resize(adv.crop_form(page_2), (2500, 3000))
   #----------------------------------
   '''     P  A  G  E   -   2      '''
   #----------------------------------

   img = get_area(form_c, 'pg2', 'part2')
   w8_content['part02']['11'] = []
   obj = checkbox(img, 'pg2-part2', txtWidth=400)
   for key in obj.keys():
       if obj[key] < 1300:
          w8_content['part02']['11'].append(key)

   img = get_area(form_c, 'pg2', 'part3')
   w8_content['part02']['14'] = []
   obj = checkbox(img, 'pg2-part3')
   for key in range(len(obj)):
       if obj[key] < 1300:
          w8_content['part03']['11'].append(key)

   img = get_area(form_c, 'pg2', 'part4')
   w8_content['part02']['17'] = []
   obj = checkbox(img, 'pg2-part4')
   for key in range(len(obj)):
       if obj[key] < 1300:
          w8_content['part04']['17'] = 'Yes'

   img = get_area(form_c, 'pg2', 'part5')
   obj = checkbox(img, 'pg2-part5')
   for key in range(len(obj)):
       if obj[key] < 1300:
          w8_content['part05']['18'] = 'Yes'

   #------------------------------------------
   '''     P  A  G  E  S -   3,4,5,6,7     '''
   #------------------------------------------

   p3_idx = {'part06:19', 'part07:21', 'part08:22', 'part09:23', 'part10:24a'}
   pg_scan(cv2.resize(adv.crop_form(page_3), (2500, 3000)), 'pg3', p3_idx)

   p4_idx = ['part10:24b', 'part10:24c', 'part10:24d', 'part11:25a', 'part11:25b', 'part11:25c', 'part12:26']
   pg_scan(cv2.resize(adv.crop_form(page_4), (2500, 3000)), 'pg4', p4_idx)

   p5_idx = ['part13:27', 'part14:28a', 'part14:28b', 'part15:29a', 'part15:29b', 'part15:29c', 'part15:29d', 'part15:29e']
   pg_scan(cv2.resize(adv.crop_form(page_5), (2500, 3000)), 'pg5', p5_idx)

   p6_idx = ['part15:29f', 'part16:30', 'part17:31', 'part18:32', 'part19:33', 'part20:34', 'part21:35']
   pg_scan(cv2.resize(adv.crop_form(page_6), (2500, 3000)), 'pg6', p6_idx)

   p7_idx = ['part22:36', 'part23:37a', 'part23:37b', 'part24:38', 'part25:39', 'part26:40a', 'part26:40b', 'part26:40c', 'part27:41']
   pg_scan(cv2.resize(adv.crop_form(page_7), (2500, 3000)), 'pg7', p7_idx)

   #----------------------------------
   '''     P  A  G  E   -   8      '''
   #----------------------------------

   model = LeNet.build(numChannels=1, imgRows=28, imgCols=28, numClasses=10, weightsPath=args["weights"] if args["load_model"] > 0 else None)
   #model = LeNet.build(numChannels=1, imgRows=28, imgCols=28, numClasses=10, weightsPath="D:\\DB_OCR\\output\\lenet_weights.hdf5")

   img_8 = cv2.imread(page_8)
   gray = cv2.cvtColor(img_8,cv2.COLOR_BGR2GRAY)
   (thresh, binImg) = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
   binImg = 255 - binImg

   kernel = np.array((2,2), np.uint8)
   binImg = cv2.erode(binImg, kernel, iterations = 2)
   binImg = cv2.dilate(binImg, kernel, iterations = 1)

   binImg = 255 - binImg
   cv2.imwrite('{}\\pg8_pre.jpg'.format(tmp_folder), binImg)

   page_8_mod = '{}\\pg8_pre.jpg'.format(tmp_folder)
   form_c = cv2.resize(adv.crop_form(page_8_mod), (2500, 3000))

   p8_idx = ['part28:43', 'part29:sign']
   pg_scan(form_c, 'pg8', p8_idx)

   outDt = ''
   formDt = form_c[1200:1280, 2100:2500]; #adv.showim(formDt) #formDt = form_c[1100:1210, 2050:2500]
   ret, thresh = cv2.threshold(~formDt, 127, 255, 0)
   image, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
   (sorted_ctrs, boundingBoxes) = adv.sort_contours(contours, method="left-to-right")
   for i, c in enumerate(sorted_ctrs):
       tmp_img = np.zeros(formDt.shape, dtype=np.uint8)
       res = cv2.drawContours(tmp_img, [c], -1, 255, cv2.FILLED)
       tmp_img = np.bitwise_and(tmp_img, ~formDt)
       ret, inverted = cv2.threshold(tmp_img, 127, 255, cv2.THRESH_BINARY_INV)
       cnt = sorted_ctrs[i]
       x, y, w, h = cv2.boundingRect(cnt)
       cropped = inverted[y:y + h, x:x + w]
       cropped_orig = formDt[y:y + h, x:x + w]
       if (w < 15 and h < 15): continue
       #adv.showim(cv2.resize(cropped, (100, 100)))
       cropped = cv2.bitwise_not(cropped)
       thresh = cv2.threshold(cropped, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
       kernel = np.ones((2,2), np.uint8)
       gray_dilation = cv2.dilate(thresh, kernel, iterations=1)
       gray_erosion = cv2.erode(gray_dilation, kernel, iterations=1)
       gray_erosion = cv2.copyMakeBorder(gray_erosion, top=15, bottom=15, left=15, right=15, borderType= cv2.BORDER_CONSTANT, value=[0,0,0])
       #print(gray_erosion.shape)
       the_img = cv2.resize(gray_erosion, (28, 28))
       the_img = np.reshape(the_img, (1,28,28,1))
       #cv2.putText(cropped_orig, str(prediction[0]), (2, 2), cv2.FONT_HERSHEY_SIMPLEX, 4, (0, 255, 0), 2)
       #adv.showim(cv2.resize(gray_erosion, (100, 100)))
       probs = model.predict(the_img)
       prediction = probs.argmax(axis=1)
       #print(prediction[0])
       outDt = outDt + str(prediction[0])

   w8_content['part29']['dt'] = outDt[:2]+'-'+outDt[3:5]+'-'+outDt[6:]

   print(json.dumps(w8_content, sort_keys=True,indent=4, separators=(',', ': ')))


import pandas as pd
import numpy as np
from ..utils import _sparse_mx_to_torch_sparse_tensor, _normalize
from tqdm.notebook import tqdm_notebook as tqdm
from scipy.sparse import coo_matrix

# =============================================================================
# GENERATE GRAPH
# =============================================================================
def extract_single_snapshot(df, day):
    """ Extract and format a single snapshot from a DataFrame for a given day.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.
    day : int
        The specific day for which to extract the snapshot.

    Returns:
    --------
    str
        A formatted string representing the snapshot for the given day.

    Notes:
    ------
    - This function extracts and formats a single snapshot of network data for a given day.
    - It is assumed that the DataFrame `df` has a 'src_ip', 'port', 'weight', and 'label' columns.

    """
    # Extract snapshots
    snapshot = df[df.interval==day]
    snapshot = snapshot.drop(columns=['interval']).values

    # Entries order: src_ip, port, weight, label
    snapshot = [','.join([
        str(x[0]), str(x[1]), str(x[2]), str(x[3])
    ]) for x in snapshot]
    snapshot = '\n'.join(snapshot)

    return snapshot

def aggregate_edges(df, gt):
    """ Aggregate edges in a DataFrame while counting packets and adding labels.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.
    gt : pd.DataFrame
        The ground truth labels for source IPs.

    Returns:
    --------
    pd.DataFrame
        An aggregated DataFrame with added packet count and labels for edges.

    Notes:
    ------
    - This function aggregates edges in the DataFrame `df`, counting packets and adding labels.
    - It assumes that the DataFrame `df` has 'src_ip', 'dst_port', and 'interval' columns.
    - The `gt` DataFrame provides ground truth labels for source IPs.

    """
    # Add one column for packets
    df['pkts'] = 1

    # Aggregate edges counting (src_ip, dst_ports)
    df = df.groupby(['src_ip', 'dst_port']).agg({
        'interval':lambda x:list(x)[0],
        'pkts':'count'
    }).reset_index()

    # Add label column for edges
    df = df.merge(gt, on='src_ip', how='left').fillna('unknown')

    return df

# =============================================================================
# TARGETED PORTS BY A SOURCE IP
# =============================================================================
def get_contacted_dst_ports(df):
    """ Get the total number of contacted destination ports per source IP.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with the total number of contacted destination ports per source IP.

    Notes:
    ------
    - This function calculates the total number of contacted destination ports per source IP.
    - It assumes that the DataFrame `df` has 'src_ip' and 'dst_port' columns.

    """
    # Total number of contacted ports
    stat = df.groupby('src_ip').agg({'dst_port':lambda x: len(set(x))})

    return stat.fillna(.0)

def get_stats_per_dst_port(df):
    """ Get general statistics of packets per destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with general statistics of packets per destination port per source IP.

    Notes:
    ------
    - This function calculates general statistics of packets per destination port per source IP.
    - It assumes that the DataFrame `df` has 'src_ip', 'dst_port', and 'interval' columns.

    """
    # General statistics of packets per destination port
    tmp = df.groupby(['src_ip', 'dst_port'])['interval']\
            .count()\
            .reset_index()\
            .rename(columns={'interval':'pkts'})
    stat = tmp.groupby('src_ip').agg({'pkts':[min, max, sum, 'mean', 'std']})

    return stat.fillna(.0)

# =============================================================================
# SOURCE IPS TARGETING A DARKNET PORT
# =============================================================================
def get_contacted_src_ips(df):
    """ Get the total number of contacted source IPs per destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with the total number of contacted source IPs per destination port.

    Notes:
    ------
    - This function calculates the total number of contacted source IPs per destination port.
    - It assumes that the DataFrame `df` has 'src_ip' and 'dst_port' columns.

    """
    # Total number of contacted ports
    stat = df.groupby('dst_port').agg({'src_ip':lambda x: len(set(x))})

    return stat.fillna(.0)

def get_stats_per_src_ip(df):
    """ Get general statistics of packets per source IP per destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with general statistics of packets per source IP per destination port.

    Notes:
    ------
    - This function calculates general statistics of packets per source IP per destination port.
    - It assumes that the DataFrame `df` has 'src_ip', 'dst_port', and 'interval' columns.
    """
    # General statistics of packets per destination port
    tmp = df.groupby(['src_ip', 'dst_port'])['interval']\
            .count()\
            .reset_index()\
            .rename(columns={'interval':'pkts'})
    stat = tmp.groupby('dst_port').agg({'pkts':[min, max, sum, 'mean', 'std']})

    return stat.fillna(.0)

# =============================================================================
# DARKNET IPS TARGETED BY A SOURCE IP
# =============================================================================
def get_contacted_dst_ips(df, dummy=False):
    """ Get the total number of contacted darknet IPs per source IP or destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.
    dummy : bool, optional
        If True, calculates the total number of contacted darknet IPs per destination port,
        by default False.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with the total number of contacted darknet IPs per source IP or
        destination port.

    Notes:
    ------
    - This function calculates the total number of contacted darknet IPs per source IP or
      destination port.
    - If `dummy` is True, it calculates the total number of contacted darknet IPs per
      destination port.
    - It assumes that the DataFrame `df` has 'src_ip', 'dst_ip', and 'dst_port' columns.
    """
    # Total number of contacted darknet IPs
    if not dummy:
        stat = df.groupby('src_ip').agg({'dst_ip':lambda x: len(set(x))})

    else:
        stat = df.groupby('dst_port').agg({'dst_ip':lambda x: len(set(x))})
        for col in stat.columns:
            stat[col].values[:] = 0

    return stat.fillna(.0)

def get_stats_per_dst_ip(df, dummy=False):
    """ Get general statistics of packets per destination IP per source IP or destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.
    dummy : bool, optional
        If True, calculates statistics per destination IP per destination port,
        by default False.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with general statistics of packets per destination IP per source IP or
        destination port.

    Notes:
    ------
    - This function calculates general statistics of packets per destination IP per source
      IP or destination port.
    - If `dummy` is True, it calculates statistics per destination IP per destination port.
    - It assumes that the DataFrame `df` has 'src_ip', 'dst_ip', 'dst_port', and
      'interval' columns.
    """
    # General statistics of packets per destination ip
    if not dummy:
        tmp = df.groupby(['src_ip', 'dst_ip'])['interval']\
                .count()\
                .reset_index()\
                .rename(columns={'interval':'pkts'})

        stat = tmp.groupby('src_ip').agg({'pkts':[min, max, sum, 'mean', 'std']})
    else:
        tmp = df.groupby(['dst_port', 'dst_ip'])['interval']\
                .count()\
                .reset_index()\
                .rename(columns={'interval':'pkts'})

        stat = tmp.groupby('dst_port').agg({'pkts':[min, max, sum, 'mean', 'std']})

        for col in stat.columns:
            stat[col].values[:] = 0

    return stat.fillna(.0)

# =============================================================================
# GENERIC PACKETS STATISTICS
# =============================================================================
def get_packet_statistics(df, by='src_ip'):
    """ Get general packet statistics per source IP or destination port.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing network data.
    by : str, optional
        The column by which to group the packet statistics ('src_ip' or 'dst_port'),
        by default 'src_ip'.

    Returns:
    --------
    pd.DataFrame
        A DataFrame with general packet statistics per source IP or destination port.

    Notes:
    ------
    - This function calculates general packet statistics per source IP or destination port.
    - The `by` parameter specifies whether to group the statistics by source IP or
      destination port.
    - It assumes that the DataFrame `df` contains relevant columns for packet statistics.
    """
    # General packets statistics
    stat = df.groupby(by).agg({
        'pck_len':[sum, min, max, 'mean', 'std'],
        'ttl':[sum, min, max, 'mean', 'std'],
        't_mss':[sum, min, max, 'mean', 'std'],
        't_win':[sum, min, max, 'mean', 'std'],
        't_ts':[sum, min, max, 'mean', 'std']
    })

    return stat.fillna(.0)

def uniform_features(df, lookup, node_type):
    """ Uniformly format and index features DataFrame based on node lookup.

    Parameters:
    -----------
    df : pd.DataFrame
        The input DataFrame containing node features.
    lookup : dict
        A dictionary mapping node names to IDs.
    node_type : str
        The type of nodes in the DataFrame (e.g., 'src_ip', 'dst_port').

    Returns:
    --------
    pd.DataFrame
        A uniformly formatted and indexed DataFrame of node features.

    Notes:
    ------
    - This function uniformly formats and indexes a DataFrame of node features based on a
      node lookup dictionary.
    - It assumes that the DataFrame `df` has a column named as specified by `node_type`.
    """
    # Concatenate and normalize
    df = pd.concat(df, axis=1)
    df = df/df.max()

    # Uniform single features dataframe
    df.columns = range(df.shape[1])
    df = df.reindex(lookup.keys()).fillna(.0).reset_index()

    # Replace nodes with node ID
    df[node_type] = df[node_type].map(lambda x: lookup[x])
    df = df.rename(columns={node_type:'index'}).set_index('index')

    return df

def generate_adjacency_matrices(flist, weighted=True):
    """ Generate adjacency matrices from a list of DataFrame files.

    Parameters:
    -----------
    flist : list
        A list of file paths, each containing a DataFrame of network data.
    weighted : bool, optional
        If True, the edges in the generated matrices will be weighted,
        by default True.

    Returns:
    --------
    list
        A list of torch sparse tensors representing the adjacency matrices.

    Notes:
    ------
    - This function generates adjacency matrices from a list of DataFrame files.
    - If `weighted` is True, the edges in the matrices will be weighted based on packet
      counts.
    """
    traces = []
    edges = []
    for fname in tqdm(flist, desc='Loading graphs'):
        data = pd.read_csv(fname, header=None,
                           names=["src", "dst", "weight", "label"])
        if not weighted:
            data['weight'] = 1

        self_loops_value = 1
        traces.append(data)

        edges.append(data[['src', 'dst', 'weight']])

    tot_nodes = max([max(edge.src.max(), edge.dst.max()) for edge in edges])+1

    indices = [np.stack([
        # Source nodes
        np.concatenate((
            edge.src.values, # Original edges
            edge.dst.values, # Symmetric edges
            np.arange(tot_nodes) # Self-loops
        )),
        # Destination nodes
        np.concatenate((
            edge.dst.values, # Original edges
            edge.src.values, # Symmetric edges
            np.arange(tot_nodes) # Self-loops
        ))]) for edge in edges]

    values = [
        np.concatenate((
            edge["weight"].values.reshape(-1,),
            edge["weight"].values.reshape(-1,),
            self_loops_value*np.ones([tot_nodes,])
        ))
        for edge in edges]

    adjs = []
    for i in tqdm(range(len(edges)), desc='Generating matrices'):
        adjs.append(
            _sparse_mx_to_torch_sparse_tensor(
                _normalize(
                    coo_matrix(
                        (values[i],
                         (indices[i][0], indices[i][1])),
                        shape=(tot_nodes, tot_nodes),
                        dtype=np.float32))))

    return adjs


from operator import attrgetter
from datetime import datetime
from ryu.app import simple_switch_13
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.lib import hub

class SimpleMonitor13(simple_switch_13.SimpleSwitch13):

    def __init__(self, *args, **kwargs):
        super(SimpleMonitor13, self).__init__(*args, **kwargs)
        self.datapaths = {}
        self.monitor_thread = hub.spawn(self._monitor)
    self.fields = {'time':'','datapath':'','in-port':'','eth_src':'','eth_dst':'','out-port':'','total_packets':0,'total_bytes':0}

    @set_ev_cls(ofp_event.EventOFPStateChange,
                [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def _state_change_handler(self, ev):
        datapath = ev.datapath
        if ev.state == MAIN_DISPATCHER:
            if datapath.id not in self.datapaths:
                self.logger.debug('register datapath: %016x', datapath.id)
                self.datapaths[datapath.id] = datapath
        elif ev.state == DEAD_DISPATCHER:
            if datapath.id in self.datapaths:
                self.logger.debug('unregister datapath: %016x', datapath.id)
                del self.datapaths[datapath.id]

    def _monitor(self):
        self.logger.info('time\tdatapath\tin-port\teth-src\teth-dst\tout-port\ttotal_packets\ttotal_bytes')
        while True:
            for dp in self.datapaths.values():
                self._request_stats(dp)
            hub.sleep(1)

    def _request_stats(self, datapath):
        self.logger.debug('send stats request: %016x', datapath.id)
        ofproto = datapath.ofproto
        parser = datapath.ofproto_parser

        req = parser.OFPFlowStatsRequest(datapath)
        datapath.send_msg(req)

        req = parser.OFPPortStatsRequest(datapath, 0, ofproto.OFPP_ANY)
        datapath.send_msg(req)

    @set_ev_cls(ofp_event.EventOFPFlowStatsReply, MAIN_DISPATCHER)
    def _flow_stats_reply_handler(self, ev):
        body = ev.msg.body

        for stat in sorted([flow for flow in body if flow.priority == 1],
                           key=lambda flow: (flow.match['in_port'],
                                             flow.match['eth_dst'])):
            #print details of flows
            self.fields['time'] = datetime.utcnow().strftime('%s')
            self.fields['datapath'] = ev.msg.datapath.id
            self.fields['in-port'] = stat.match['in_port']
            self.fields['eth_src'] = stat.match['eth_src']
            self.fields['eth_dst'] = stat.match['eth_dst']
            self.fields['out-port'] = stat.instructions[0].actions[0].port
            self.fields['total_packets'] = stat.packet_count
            self.fields['total_bytes'] = stat.byte_count

            self.logger.info('data\t%s\t%x\t%x\t%s\t%s\t%x\t%d\t%d',self.fields['time'],self.fields['datapath'],self.fields['in-port'],self.fields['eth_src'],self.fields['eth_dst'],self.fields['out-port'],self.fields['total_packets'],self.fields['total_bytes'])


#!/usr/bin/python3
import requests
import json
from dateutil import parser
import itertools
from collections import deque
import glob
import os
from dotenv import load_dotenv
load_dotenv()

dateOld = []
confirmedOld = []
recoveredOld = []
deathsOld = []
treatmentOld = []
newConfirmedList = []
newRecoveredList = []
newDeathsList = []
newTreatmentList = []
avg7ConfirmedList = []
avg7RecoveredList = []
avg7DeathsList = []
avg7TreatmentList = []
genderOld = []
ageOld = []
ageConfirmedOld = []
ageDeathsOld = []
lastUpdatedOld = ''
totalOfficialStats = []
provinces = {}

token = ''
URL = 'https://api.corona-dz.live'

def login():
    data = {
        'email': '{}'.format(os.getenv('API_EMAIL')),
        'password': '{}'.format(os.getenv('API_PASSWORD'))
    }

    try:
        response = requests.post(URL+'/login', data=data)
        response.raise_for_status()
        return response.json()['token']
    except requests.exceptions.RequestException as e:
        raise SystemExit(e)

def addCountry():
    head = {'Authorization': 'Bearer {}'.format(token)}

    for i in range(len(dateOld)):
        data = {
            'date': parser.parse(dateOld[i]).date(),
            'confirmed': confirmedOld[i],
            'recovered': recoveredOld[i],
            'deaths': deathsOld[i],
            'treatment': treatmentOld[i],
            'newConfirmed': newConfirmedList[i],
            'newRecovered': newRecoveredList[i],
            'newDeaths': newDeathsList[i],
            'newTreatment': newTreatmentList[i],
            'avg7Confirmed': avg7ConfirmedList[i],
            'avg7Recovered': avg7RecoveredList[i],
            'avg7Deaths': avg7DeathsList[i],
            'avg7Treatment': avg7TreatmentList[i]
        }
        # print(data)
        response = requests.post(URL+'/country', data=data, headers=head)
        if(response.json()):
            print(response.json())
        # break

def addAge():
    head = {'Authorization': 'Bearer {}'.format(token)}

    for i in range(len(ageOld)):
        data = {
            'date': parser.parse(lastUpdatedOld).date(),
            'label': ageOld[i],
            'confirmed': ageConfirmedOld[i],
            'deaths': ageDeathsOld[i]
        }
        # print(data)
        response = requests.post(URL+'/country/age', data=data, headers=head)
        if(response.json()):
            print(response.json())

def addGender():
    head = {'Authorization': 'Bearer {}'.format(token)}

    data = {
        'date': parser.parse(lastUpdatedOld).date(),
        'male': genderOld[0],
        'female': genderOld[1],
    }
    # print(data)
    response = requests.post(URL+'/country/gender', data=data, headers=head)
    if(response.json()):
        print(response.json())

def addProvince():
    head = {'Authorization': 'Bearer {}'.format(token)}

    for p_id, p in provinces.items():
        data = {
            'date': parser.parse(lastUpdatedOld).date(),
            'provinceId': p_id,
            'confirmed': p['confirmed'],
            'recovered': p['recovered'],
            'deaths': p['deaths'],
            'newConfirmed': p['new_confirmed'],
            'newRecovered': p['new_recovered'],
            'newDeaths': p['new_deaths'],
            'avg7Confirmed': p['avg7_confirmed'],
            'avg7Recovered': p['avg7_recovered'],
            'avg7Deaths': p['avg7_deaths'],
            'firstReported': p['reported'],
            'lastReported': p['last_reported']
        }
        #print(data)
        response = requests.post(URL+'/province', data=data, headers=head)
        if(response.json()):
            print(response.json())
        # break

def readData(f, country=False):
    global dateOld, confirmedOld, recoveredOld, deathsOld, treatmentOld, newConfirmedList, newRecoveredList, newDeathsList, newTreatmentList
    global avg7ConfirmedList, avg7RecoveredList, avg7DeathsList, avg7TreatmentList # , newConfirmedOld, newRecoveredOld, newDeathsOld,
    global provinces, genderOld, ageOld, ageConfirmedOld, ageDeathsOld, lastUpdatedOld

    with open(f) as f:
        for line in f:
            if(line.startswith('export const date')):
                dateOld = line[line.find("[")+1:line.find("]")].replace("'", "").split(', ')
            elif(line.startswith('export const confirmed')):
                confirmedOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const recovered')):
                recoveredOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const deaths')):
                deathsOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const treatment')):
                treatmentOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const genderData')):
                genderOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const age =')):
                ageOld = line[line.find("[")+1:line.find("]")].split(',')
            elif(line.startswith('export const ageConfirmedData')):
                ageConfirmedOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const ageDeathsData')):
                ageDeathsOld = list(map(int, line[line.find("[")+1:line.find("]")].split(',')))
            elif(line.startswith('export const lastUpdated')):
                lastUpdatedOld = line[line.find("'")+1:line.find("'", -2)]
            elif(line.startswith('  "')):
                p = json.loads('{'+line[line.find("{")+1:line.find("}")].replace(': ', '": "').replace(', ', '", "').replace('""', '"').replace(' id', ' "id')+'}')
                pId = int(p['id'])
                provinces[pId] = {}
                provinces[pId]['name'] = line[line.find('"')+1:line.find('":')]
                provinces[pId]['confirmed'] = p['confirmed']
                provinces[pId]['recovered'] = p['recovered']
                provinces[pId]['deaths'] = p['deaths']
                provinces[pId]['new_confirmed'] = p['new_confirmed']
                provinces[pId]['new_recovered'] = p['new_recovered']
                provinces[pId]['new_deaths'] = p['new_deaths']
                provinces[pId]['avg7_confirmed'] = p['avg7_confirmed']
                provinces[pId]['avg7_recovered'] = p['avg7_recovered']
                provinces[pId]['avg7_deaths'] = p['avg7_deaths']
                provinces[pId]['reported'] = p['reported']
                provinces[pId]['last_reported'] = p['last_reported']
    if country:
        newConfirmedList = getDailyData(confirmedOld)
        newRecoveredList = getDailyData(recoveredOld)
        newDeathsList = getDailyData(deathsOld)
        newTreatmentList = getDailyData(treatmentOld)
        avg7ConfirmedList = getAverageDailyData(newConfirmedList, 7)
        avg7RecoveredList = getAverageDailyData(newRecoveredList, 7)
        avg7DeathsList = getAverageDailyData(newDeathsList, 7)
        avg7TreatmentList = getAverageDailyData(newTreatmentList, 7)
        # #print(avg7ConfirmedList, len(avg7ConfirmedList))
    # print(len(newTreatmentList), len(newConfirmedList), len(avg7ConfirmedList))

def getDailyData(dataType):
    dailyData = []
    for i in range(len(dataType)):
        if i == 0:
            dailyData.append(dataType[i])
        else:
            dailyData.append(dataType[i] - (dataType[i-1]))
    return dailyData

def getAverageDailyData(dailyData, days):
    averageDailyData = []
    it = iter([0] * (days-1) + dailyData)
    d = deque(itertools.islice(it, days-1))
    d.appendleft(0)
    s = sum(d)
    for elem in it:
        s += elem - d.popleft()
        d.append(elem)
        averageDailyData.append(round(s / days, 2))

    return averageDailyData

def importData():
    filesList = sorted(glob.glob("data-hist/*.js"))
    for f in filesList:
        print(f)
        readData(f)
        addAge()
        addGender()
        addProvince()
    # add country stats only from the newset file, since it containts all the data
    readData(filesList[-1], country=True)
    addCountry()

def main():
    global token
    token = login()
    importData()

if __name__ == '__main__':
    main()


from page_objects.home_page import HomePage

class HomeTest(HomePage):

    def setUp(self):
        # Call the parent BaseCase class setup method
        super().setUp()
        print("RUNNING BEFORE EACH TEST")

        # LOGIN
        self.login()

        # open home page
        self.open_page()

    def tearDown(self):
        print("RUNNING AFTER EACH TEST")

        # Logout
        self.open("https://practice.automationbro.com/my-account")
        self.click(".woocommerce-MyAccount-content a[href*=logout]")
        self.assert_element_visible("button[name=login]")

        super().tearDown()

    def test_home_page(self):
        # assert page title
        self.assert_title("Practice E-Commerce Site – Automation Bro")

        # assert logo
        self.assert_element(HomePage.logo_icon)

        # click on the get started button and assert the url
        self.click(HomePage.get_started_btn)
        get_started_url = self.get_current_url()
        self.assert_equal(get_started_url, "https://practice.automationbro.com/#get-started")
        self.assert_true("get-started" in get_started_url)

        # get the text of the header and assert the value
        self.assert_text("Think different. Make different.", HomePage.heading_text)

        # Exercise: scroll to bottom and assert the copyright text
        self.scroll_to_bottom()
        self.assert_text("Copyright © 2021 Automation Bro", HomePage.copyright_text)

    def test_menu_links(self):
        expected_links = ['Home', 'About', 'Shop', 'Blog', 'Contact', 'My account']

        # find menu links elements
        menu_links_el = self.find_elements(HomePage.menu_links)

        # loop through our menu links
        for idx, link_el in enumerate(menu_links_el):
            # print(idx, link_el.text)
            self.assertEqual(expected_links[idx], link_el.text)


import random
import shutil
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import Iterator
from datetime import timedelta
from io import BytesIO
from pathlib import Path

import lance
import lancedb
import pyarrow as pa
import shortuuid
from PIL import Image
from tqdm.auto import tqdm

from pixano.data import Dataset, DatasetInfo, Fields
from pixano.utils import estimate_size

class Importer(ABC):
    """Dataset Importer class

    Attributes:
        info (DatasetInfo): Dataset information
    """

    def __init__(
        self,
        name: str,
        description: str,
        tables: dict[str, list],
        splits: list[str],
    ):
        """Initialize Importer

        Args:
            name (str): Dataset name
            description (str): Dataset description
            tables (dict[str, list]): Dataset fields
            splits (list[str]): Dataset splits
        """

        # Dataset info
        self.info = DatasetInfo(
            id=shortuuid.uuid(),
            name=name,
            description=description,
            estimated_size="N/A",
            num_elements=0,
            preview=None,
            splits=splits,
            tables=tables,
            categories=[],
        )

    def create_json(
        self,
        import_dir: Path,
        ds_tables: dict[str, dict[str, lancedb.db.LanceTable]],
    ):
        """Create dataset spec.json

        Args:
            import_dir (Path): Import directory
            ds_tables (dict[str, dict[str, lancedb.db.LanceTable]]): Dataset tables
        """

        self.info.num_elements = len(ds_tables["main"]["db"])
        self.info.estimated_size = estimate_size(import_dir)

        # Save DatasetInfo
        with tqdm(desc="Creating dataset info file", total=1) as progress:
            self.info.save(import_dir)
            progress.update(1)

    def create_preview(
        self,
        import_dir: Path,
        ds_tables: dict[str, dict[str, lancedb.db.LanceTable]],
    ):
        """Create dataset preview image

        Args:
            import_dir (Path): Import directory
            ds_tables (dict[str, dict[str, lancedb.db.LanceTable]]): Dataset tables
        """

        # Get list of image fields
        if "media" in ds_tables:
            if "image" in ds_tables["media"]:
                image_table = ds_tables["media"]["image"]
                if len(image_table) > 0:
                    image_fields = [
                        field.name for field in image_table.schema if field.name != "id"
                    ]
                    with tqdm(desc="Creating dataset thumbnail", total=1) as progress:
                        tile_w = 64
                        tile_h = 64
                        preview = Image.new("RGB", (4 * tile_w, 2 * tile_h))
                        for i in range(8):
                            field = image_fields[i % len(image_fields)]
                            item_id = random.randrange(len(image_table))
                            item = image_table.to_lance().take([item_id]).to_pylist()[0]
                            with Image.open(BytesIO(item[field].preview_bytes)) as im:
                                preview.paste(
                                    im,
                                    ((i % 4) * tile_w, (int(i / 4) % 2) * tile_h),
                                )
                        preview.save(import_dir / "preview.png")
                        progress.update(1)

    @abstractmethod
    def import_rows(
        self,
        input_dirs: dict[str, Path],
    ) -> Iterator:
        """Process dataset rows for import

        Args:
            input_dirs (dict[str, Path]): Input directories

        Yields:
            Iterator: Processed rows
        """

    def import_dataset(
        self,
        input_dirs: dict[str, Path],
        import_dir: Path,
        copy: bool = True,
    ) -> Dataset:
        """Import dataset to Pixano format

        Args:
            input_dirs (dict[str, Path]): Input directories
            import_dir (Path): Import directory
            copy (bool, optional): True to copy files to the import directory, False to move them. Defaults to True.

        Returns:
            Dataset: Imported dataset
        """

        # Check input directories
        for source_path in input_dirs.values():
            if not source_path.exists():
                raise FileNotFoundError(f"{source_path} does not exist.")
            if not any(source_path.iterdir()):
                raise FileNotFoundError(f"{source_path} is empty.")

        # Connect to dataset
        import_dir.mkdir(parents=True, exist_ok=True)
        ds = lancedb.connect(import_dir)

        # Initialize dataset tables
        ds_tables: dict[str, dict[str, lancedb.db.LanceTable]] = defaultdict(dict)
        ds_batches: dict[str, dict[str, list]] = defaultdict(dict)

        # Create tables
        for table_group, tables in self.info.tables.items():
            for table in tables:
                ds_tables[table_group][table["name"]] = ds.create_table(
                    table["name"],
                    schema=Fields(table["fields"]).to_schema(),
                    mode="overwrite",
                )
                ds_batches[table_group][table["name"]] = []
        save_batch_size = 1024

        # Add rows to tables
        for rows in tqdm(
            self.import_rows(input_dirs),
            desc="Importing dataset",
        ):
            for table_group, tables in self.info.tables.items():
                for table in tables:
                    # Store rows in a batch
                    ds_batches[table_group][table["name"]].extend(
                        rows[table_group][table["name"]]
                    )
                    # If batch reaches 1024 rows, store in table
                    if len(ds_batches[table_group][table["name"]]) >= save_batch_size:
                        pa_batch = pa.Table.from_pylist(
                            ds_batches[table_group][table["name"]],
                            schema=Fields(table["fields"]).to_schema(),
                        )
                        lance.write_dataset(
                            pa_batch,
                            uri=ds_tables[table_group][table["name"]].to_lance().uri,
                            mode="append",
                        )
                        ds_batches[table_group][table["name"]] = []

        # Store final batches
        for table_group, tables in self.info.tables.items():
            for table in tables:
                if len(ds_batches[table_group][table["name"]]) > 0:
                    pa_batch = pa.Table.from_pylist(
                        ds_batches[table_group][table["name"]],
                        schema=Fields(table["fields"]).to_schema(),
                    )
                    lance.write_dataset(
                        pa_batch,
                        uri=ds_tables[table_group][table["name"]].to_lance().uri,
                        mode="append",
                    )
                    ds_batches[table_group][table["name"]] = []

        # Optimize and clear creation history
        for tables in ds_tables.values():
            for table in tables.values():
                table.to_lance().optimize.compact_files()
                table.to_lance().cleanup_old_versions(older_than=timedelta(0))

        # Refresh tables
        for table_group, tables in self.info.tables.items():
            for table in tables:
                ds_tables[table_group][table["name"]] = ds.open_table(table["name"])

        # Raise error if generated dataset is empty
        if len(ds_tables["main"]["db"]) == 0:
            raise FileNotFoundError(
                "Generated dataset is empty. Please make sure that the paths to your media files are correct, and that they each contain subfolders for your splits."
            )

        # Copy or move media directories
        if "media" in ds_tables:
            if copy:
                for table in tqdm(
                    ds_tables["media"].values(), desc="Copying media directories"
                ):
                    for field in table.schema:
                        if field.name in input_dirs:
                            field_dir = import_dir / "media" / field.name
                            field_dir.mkdir(parents=True, exist_ok=True)
                            if input_dirs[field.name] != field_dir:
                                shutil.copytree(
                                    input_dirs[field.name],
                                    field_dir,
                                    dirs_exist_ok=True,
                                )
            else:
                for table in tqdm(
                    ds_tables["media"].values(), desc="Moving media directories"
                ):
                    for field in table.schema:
                        if field.name in input_dirs:
                            field_dir = import_dir / "media" / field.name
                            if input_dirs[field.name] != field_dir:
                                input_dirs[field.name].rename(field_dir)

        # Create spec.json
        self.create_json(import_dir, ds_tables)

        # Create preview.png
        self.create_preview(import_dir, ds_tables)

        return Dataset(import_dir)


# Makes 3 different coffee recipes: Expresso, Latte, Cappuccino
# The machine holds the following resources: 300 ml water, 200 ml coffee, and 100 g sugar
# Cost - using American coins quarter, dime, nickel, and penny
# Print a report about the resources left

MENU = {
    "espresso": {
        "ingredients": {
            "water": 50,
            "coffee": 18,
        },
        "cost": 1.5,
    },
    "latte": {
        "ingredients": {
            "water": 200,
            "milk": 150,
            "coffee": 24,
        },
        "cost": 2.5,
    },
    "cappuccino": {
        "ingredients": {
            "water": 250,
            "milk": 100,
            "coffee": 24,
        },
        "cost": 3.0,
    }
}

resources = {
    "water": 300,
    "milk": 200,
    "coffee": 100,
}

#4 Check if there are sufficient resources left to make each order as it is
#placed. If not, tell user their order cannot be made. If so, proceed to
#payment.
def enough_resources(need_ingredients):
	for item in need_ingredients:
		if need_ingredients[item] > resources[item]:
			print("Sorry there is not enough {item}. ")
			return False
	return True

#5 Process payment (payment comes in the form of coins only)
def receive_payment():
	print("You selected {user_order}. Your total is ${profit}. Please insert coins.")
	total = int(input("How many quarters?: ")) * 0.25
	total += int(input("How many dimes?: ")) * 0.10
	total += int(input("How many nickels?: ")) * 0.05
	total += int(input("How many pennies?: ")) * 0.01
	return total

#6 Check if the user entered enough money to purchase their coffee order.
def check_payment(coins_inserted, cost):
	if coins_inserted >= cost:
		change = round(coins_inserted - cost, 2)
		print(f"Your change is ${change}.")
		global profit
		profit += cost
		return True
	else:
		print("I'm sorry that is not enough money. Your money has been refunded.")
		return False

#7 Make coffee order and deduct what is used from the available resources as well as add funds to the money category
#8 Tell the user "Here is your (drink name). Enjoy!"
def make_order(drink_name, order_ingredients):
	for item in order_ingredients:
		resources[item] -= order_ingredients[item]
		print("Here is your {drink_name}. ☕ Enjoy!")

#1 Prompt user by asking "What would you like? Expresso, Latte, or Cappuccino: "
#2 Turn off Coffee Machine by entering "off" to the prompt
#3 Print report of resources currently in the machine (i.e. water, milk, coffee, and money)
is_on = True

while is_on:
	user_order = input("What would you like? expresso, latte or cappucino: ")
	if user_order == "off":
		is_on = False
	elif user_order == "report":
		print(f"Water: {resources['water']}ml")
		print(f"Milk: {resources['milk']}ml")
		print(f"Coffee: {resources['coffee']}g")
		print(f"Money: ${profit}")
	else:
		drink = MENU[user_order]
		if enough_resources(drink["ingredients"]):
			payment = process_coins()
			if check_payment(payment, drink["cost"]):
				make_order(user_order, drink["ingredients"])


import numpy
from spinn_front_end_common.utilities.constants import BYTES_PER_WORD

# Address to indicate that the synaptic region is unused
SYN_REGION_UNUSED = 0xFFFFFFFF

class GeneratorData(object):
    """
    Data for each connection of the synapse generator.
    """
    __slots__ = [
        "__data"
    ]

    BASE_SIZE = 11 * BYTES_PER_WORD

    def __init__(
            self, synaptic_matrix_offset, delayed_synaptic_matrix_offset,
            app_edge, synapse_information, max_row_info,
            max_pre_atoms_per_core, max_post_atoms_per_core):
        # Offsets are used in words in the generator, but only
        # if the values are valid
        if synaptic_matrix_offset is not None:
            synaptic_matrix_offset //= BYTES_PER_WORD
        else:
            synaptic_matrix_offset = SYN_REGION_UNUSED
        if delayed_synaptic_matrix_offset is not None:
            delayed_synaptic_matrix_offset //= BYTES_PER_WORD
        else:
            delayed_synaptic_matrix_offset = SYN_REGION_UNUSED

        # Take care of Population views
        pre_lo = 0
        pre_hi = synapse_information.n_pre_neurons - 1
        if synapse_information.prepop_is_view:
            indexes = synapse_information.pre_population._indexes
            pre_lo = indexes[0]
            pre_hi = indexes[-1]
        post_lo = 0
        post_hi = synapse_information.n_post_neurons - 1
        if synapse_information.postpop_is_view:
            indexes = synapse_information.post_population._indexes
            post_lo = indexes[0]
            post_hi = indexes[-1]

        # Get objects needed for the next bit
        connector = synapse_information.connector
        synapse_dynamics = synapse_information.synapse_dynamics

        # Create the data needed
        self.__data = list()
        self.__data.append(numpy.array([
            pre_lo, pre_hi, post_lo, post_hi,
            synapse_information.synapse_type,
            synapse_dynamics.gen_matrix_id,
            connector.gen_connector_id,
            connector.gen_weights_id(synapse_information.weights),
            connector.gen_delays_id(synapse_information.delays)
            ], dtype=numpy.uint32))
        self.__data.append(synapse_dynamics.gen_matrix_params(
            synaptic_matrix_offset, delayed_synaptic_matrix_offset, app_edge,
            synapse_information, max_row_info, max_pre_atoms_per_core,
            max_post_atoms_per_core))
        self.__data.append(connector.gen_connector_params())
        self.__data.append(connector.gen_weights_params(
            synapse_information.weights))
        self.__data.append(connector.gen_delay_params(
            synapse_information.delays))

    @property
    def size(self):
        """
        The size of the generated data, in bytes.

        :rtype: int
        """
        return sum(len(i) for i in self.__data) * BYTES_PER_WORD

    @property
    def gen_data(self):
        """
        The data to be written for this connection.

        :rtype: list(~numpy.ndarray(~numpy.uint32))
        """
        return self.__data


import re

import scrapy
from scrapy.http import HtmlResponse

from ..db import load_prefecture_href
from ..items import BlockInfo

def load_start_urls() -> list[str]:
    hrefs = load_prefecture_href()
    return [
        f"https://www.data.jma.go.jp/stats/etrn/select/{href}"
        for href in hrefs
    ]

class BlockSpider(scrapy.Spider):
    name = "block"
    allowed_domains = ["www.data.jma.go.jp"]
    start_urls = load_start_urls()

    def parse(self, response):
        if type(response) is not HtmlResponse:
            return
        res: HtmlResponse = response
        tags = res.xpath("//map[@name='point']//area")
        alts = tags.xpath("@alt").extract()
        hrefs = tags.xpath("@href").extract()
        prec_nums = [
            re.sub(r".*prec_no=([0-9]+).*", r"\1", href)
            for href in hrefs
        ]
        block_nums = [
            re.sub(r".*block_no=([0-9]+).*", r"\1", href)
            for href in hrefs
        ]
        for alt, href, prec_num, block_num in zip(alts, hrefs, prec_nums, block_nums):
            if re.match(r"^[0-9]+$", block_num) is not None:
                yield BlockInfo(name=alt, href=href,
                                number=block_num, prefecture_number=prec_num)


class Solution:
    def nthSuperUglyNumber(self, n: int, primes: List[int]) -> int:
        val = [1] * len(primes)
        index = [0] * len(primes)
        ugly = [0] * n
        next = 1
        for i in range(n):
            ugly[i] = next
            next = sys.maxsize

            for j in range(len(primes)):
                if ugly[i] == val[j]:
                    val[j] = ugly[index[j]] * primes[j]
                    index[j] += 1
                next = min(next, val[j])
        return ugly[-1]


from discord import Embed, Permissions
import pytest

from mmobot.test.mock import MockMember, MockTextChannel

DEFAULT_PERMISSIONS = 0

@pytest.fixture
def member():
    return MockMember(1, 'member1')

@pytest.fixture
def channel():
    return MockTextChannel(2, 'channel_name')

@pytest.fixture
def read_write_permissions():
    return Permissions(
        DEFAULT_PERMISSIONS,
        read_messages=True,
        send_messages=True
    )

def test_MockTextChannel_initialize(channel):
    assert channel is not None
    assert channel.name is not None
    assert channel.name == 'channel_name'
    assert channel.messages is not None
    assert len(channel.messages) == 0

def test_MockTextChannel_permissionsFor_default_read_write(member, channel):
    permissions = channel.permissions_for(member)
    assert permissions is not None
    assert permissions.read_messages is False
    assert permissions.send_messages is False

def test_MockTextChannel_permissionsFor_default_no_permissions(member, read_write_permissions):
    read_write_channel = MockTextChannel(1, 'channel_name', permissions=read_write_permissions)
    permissions = read_write_channel.permissions_for(member)
    assert permissions is not None
    assert permissions.read_messages is True
    assert permissions.send_messages is True

@pytest.mark.asyncio
async def test_MockTextChannel_setPermissions_to_read_write(member, channel):
    await channel.set_permissions(member, read_messages=True, send_messages=True)
    permissions = channel.permissions_for(member)
    assert permissions.read_messages is True
    assert permissions.send_messages is True
    assert member in channel.members

@pytest.mark.asyncio
async def test_MockTextChannel_setPermissions_to_no_permissions(member, channel):
    await channel.set_permissions(member, read_messages=False, send_messages=False)
    permissions = channel.permissions_for(member)
    assert permissions.read_messages is False
    assert permissions.send_messages is False
    assert member not in channel.members

@pytest.mark.asyncio
async def test_MockTextChannel_send(channel):
    await channel.send('TEST MESSAGE')
    assert len(channel.messages) == 1
    assert channel.messages[0] == 'TEST MESSAGE'

@pytest.mark.asyncio
async def test_MockChannel_sendWithEmbed(channel):
    embed = Embed(
        title='Embed title',
        description='description'
    )
    embed.add_field(name='field1', value='value1')
    await channel.send('', embed=embed)
    assert len(channel.messages) == 1
    expected_message = '<title>Embed title</title>\n<desc>description</desc>\n'
    expected_message += '<field>field1:value1</field>\n'
    assert channel.messages[0] == expected_message


import os
import commands
from testrunner import PluginsTestCase as TestCaseBase
from plugins import check_disk_and_inode as checkDisk

class TestDisk(TestCaseBase):
    def mock_osaccess(self, path=None, osflag=None):
        return True

    def mock_getstatusoutput(self, i):
        out = [
            "Filesystem      Size  Used Avail Use% Mounted on",
            "/dev/sda1       290G  174G  102G  64% /",
            "/dev/sdb1        3.9G     0  3.9G   0% /var",
            "/dev/sdc1       290G  200G  100G  40% /mnt1",
            "/dev/mapper/vg_test1-lv_one "
            "35852      1774     32257   85% /lvm1",
            "-                          994         1       994   1% /dev",
            "/dev/mapper/vg_demonode2-lv1 "
            "50G   11G   36G  24% /mnt/lv4",
            "/dev/vda1             485M   36M  424M   8% /mnt/abc1",
            "/dev/mapper/vg_demonode2-lv_2 "
            "5.5G  4.3G  933M  83% /mnt/abc2",
            "none                     0     0     0   -  "
            "/proc/sys/fs/binfmt_misc",
            "10.70.43.190:/exports/abc    50G   11G   36G  24% /mnt",
            "10.70.43.190:vol3      50G   11G   36G  24% /mnt/vol3",
            "10.70.43.190:vol2  52846MB   23228MB   26934MB  47% /mnt/vol2"]

        if type(i) is str:
            disk = i.split()[-1]
            for diskDetail in out:
                if diskDetail.find(disk) >= 0:
                    return 0, diskDetail
        else:
            return 0, out[0] + out[i]
        return 0, ""

    def mock_open(self, fileName):
        out = ["/dev/mapper/vg_demonode2-lv_1 /mnt/abc1 ext4 rw 0 0",
               "proc /proc proc rw 0 0",
               "sysfs /sys sysfs rw 0 0",
               "devpts /dev/pts devpts rw,gid=5,mode=620 0 0",
               "tmpfs /dev/shm tmpfs rw 0 0",
               "/dev/sda1 /boot ext4 rw 0 0",
               "/dev/mapper/vg_demonode2-lv_2 /mnt/abc2 ext4 rw 0 0",
               "none /proc/sys/fs/binfmt_misc binfmt_misc rw 0 0",
               "sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw 0 0",
               "10.70.43.190:/exports/abc /mnt nfs rw,vers=4,addr=10.70.43.190"
               ",clientaddr=10.70.43.190 0 0",
               "10.70.43.190:vol2 /mnt/vol2 nfs rw,addr=10.70.43.190 0 0",
               "10.70.43.190:vol3 /mnt/vol3 nfs rw,addr=10.70.43.190 0 0"]

        class file():
            def readlines(self):
                return out

            def close(self):
                return None
        return file()

    def test_getUsageAndFree(self):
        commands.getstatusoutput = self.mock_getstatusoutput
        os.access = self.mock_osaccess
        disk = checkDisk.getUsageAndFree(1, "", 80, 90, "")
        self.assertEqual(disk['usePcent'], 64)
        self.assertEqual(disk['availPcent'], 36)
        self.assertEqual(disk['used'], 174)
        self.assertEqual(disk['avail'], 102)
        self.assertEqual(disk['path'], '/')

        disk = checkDisk.getUsageAndFree(2, "", 80, 90, "")
        self.assertEqual(disk['usePcent'], 0)
        self.assertEqual(disk['availPcent'], 100)
        self.assertEqual(disk['used'], 0)
        self.assertEqual(disk['avail'], 3.0)
        self.assertEqual(disk['path'], '/var')

        disk = checkDisk.getUsageAndFree(3, "", 80, 90, "")
        self.assertEqual(disk['usePcent'], 40)
        self.assertEqual(disk['availPcent'], 60)
        self.assertEqual(disk['used'], 200)
        self.assertEqual(disk['avail'], 100)
        self.assertEqual(disk['path'], '/mnt1')

        disk = checkDisk.getUsageAndFree(4, "", 80, 90, "")
        self.assertEqual(disk['usePcent'], 85)
        self.assertEqual(disk['availPcent'], 15)
        self.assertEqual(disk['used'], 1774)
        self.assertEqual(disk['avail'], 32257)
        self.assertEqual(disk['path'], '/lvm1')

    def test_getMounts(self):
        checkDisk.open = self.mock_open
        mounts = checkDisk.getMounts("/", [])
        self.assertEqual(mounts[0], "/dev/mapper/vg_demonode2-lv_1")
        self.assertEqual(mounts[1], "/dev/sda1")
        self.assertEqual(mounts[2], "/dev/mapper/vg_demonode2-lv_2")

        mounts = checkDisk.getMounts("/dev/mapper", [])
        self.assertEqual(mounts[0], "/dev/mapper/vg_demonode2-lv_1")
        self.assertEqual(mounts[1], "/dev/mapper/vg_demonode2-lv_2")

        mounts = checkDisk.getMounts("", [])
        self.assertEqual(len(mounts), 12)

    def test_diskUsage(self):
        commands.getstatusoutput = self.mock_getstatusoutput
        checkDisk.open = self.mock_open
        os.access = self.mock_osaccess
        mounts = checkDisk.getMounts("/", [])

        self.assertEqual(checkDisk.showDiskUsage(80,
                                                 90,
                                                 [mounts[1]],
                                                 True,
                                                 usage='BGB'),
                         (0, '64.0% used (174.0BGB out of 290.0BGB)\n'
                          ':mount(s): (/dev/sda1=/)',
                          ['/=174.0BGB;232.0;261.0;0;290.0 '
                           '/=174.0;139.2;156.6;0;290.0']))

        self.assertEqual(checkDisk.showDiskUsage(80,
                                                 90,
                                                 [mounts[1]],
                                                 True),
                         (0, '64.0% used (174.0GB out of 290.0GB)\n'
                          ':mount(s): (/dev/sda1=/)',
                          ['/=64.00%;80;90;0;290.0GB '
                           '/=64.00%;80;90;0;290.0']))

        self.assertEqual(checkDisk.showDiskUsage(80,
                                                 90,
                                                 [mounts[1]], True),
                         (0, '64.0% used (174.0GB out of 290.0GB)\n'
                          ':mount(s): (/dev/sda1=/)',
                          ['/=64.00%;80;90;0;290.0GB '
                           '/=64.00%;80;90;0;290.0']))

        self.assertEqual(checkDisk.showDiskUsage(80,
                                                 90,
                                                 ["/mnt/vol2"], True),
                         (0, '47.0% used (22.68359375TB out of '
                          '51.607421875TB)\n'
                          ':mount(s): (10.70.43.190:vol2=/mnt/vol2)',
                          ['/mnt/vol2=47.00%;80;90;0;52846.0GB '
                           '/mnt/vol2=47.00%;80;90;0;52846.0']))

        self.assertEqual(checkDisk.showDiskUsage(10,
                                                 20,
                                                 ["/mnt/vol2"], True),
                         (2, '47.0% used (22.68359375TB out of '
                          '51.607421875TB)\n'
                          ':mount(s): (10.70.43.190:vol2=/mnt/vol2)',
                          ['/mnt/vol2=47.00%;10;20;0;52846.0GB '
                           '/mnt/vol2=47.00%;10;20;0;52846.0']))

        # negative test
        self.assertEqual(checkDisk.showDiskUsage(1,
                                                 100,
                                                 ["/mnt/vol2"], True),
                         (1, '47.0% used (22.68359375TB out of '
                          '51.607421875TB)\n'
                          ':mount(s): (10.70.43.190:vol2=/mnt/vol2)',
                          ['/mnt/vol2=47.00%;1;100;0;52846.0GB '
                           '/mnt/vol2=47.00%;1;100;0;52846.0']))


import arcpy, os, re
from arcpy.sa import *

# Check out any necessary licenses.
arcpy.CheckOutExtension("spatial")

scratchWorkspace = r"FWS_ScratchWorkspace.gdb"

# set up environments
arcpy.env.extent = "1272796.01889252 837937.189808734 1293290.19762114 851783.705021468" # temporary
arcpy.env.workspace = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\FWS_HabConScriptTesting"
arcpy.env.scratchWorkspace = os.path.join(arcpy.env.workspace,scratchWorkspace)
# check if the scratch geodatabase exists
if arcpy.Exists(arcpy.env.scratchWorkspace):
    # if it exists, delete it
    arcpy.Delete_management(arcpy.env.scratchWorkspace)
    print("Geodatabase deleted.")
else:
    # if it doesn't exist, print a message
    print("Geodatabase does not exist.")

arcpy.management.CreateFileGDB(arcpy.env.workspace, scratchWorkspace)

HabitatCondition_gdb = r"FWS_HabConScriptTesting.gdb"
arcpy.env.overwriteOutput =  True

# Input Variables
hexgrid = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\SE_FWS_HabitatCondition\SE_FWS_HabitatCondition.gdb\GenerateTessellation100acres"
combined_EVT_tif = r"S:\Projects\_Workspaces\Jordana_Anderson\SE_USFWS\FWS_SE_Condition\FWS_SE_Condition_testing\FWS_SE_Condition_testing.gdb\combined_EVT_Clip"
hexclip = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\FWS_HabConScriptTesting\FWS_HabConScriptTesting.gdb\hexclip"
dataLCM = r"S:\Data\NatureServe\Landscape_Condition\Americas_N_LCM_Cat100.tif"
dataRuderal = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\SE_FWS_HabitatCondition\Ruderal_5cell_scaled.tif"
dataFireDep = r"S:\Data\External\LANDFIRE_Fire_Departure\LF2020_VDep_220_CONUS\LF2020_VDep_220_CONUS\Tif\LC20_VDep_220.tif"

templateInvasiveSource = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\FWS_HabConScriptTesting\fwshabcondition\template_InvasiveSource.lyrx"
templateLCM = r"S:\Projects\USFWS\SE_FWS_Habitat_2022\FWS_HabConScriptTesting\fwshabcondition\template_LCM.lyrx"

outExtractByMask = ExtractByMask(combined_EVT_tif, hexclip)  # THIS WILL LIKELY HAVE TO BE MODIFED FOR THE FULL SCRIPT!!!!
outExtractByMask1 = ExtractByAttributes(outExtractByMask, '"Value_1" <> 0')
outExtractByMask1.save("EVTclip.tif")

value_list = []
with arcpy.da.SearchCursor("EVTclip.tif", ["Value","NatureServ"]) as cursor:
    for row in cursor:
        value_list.append(row)

col_idx = 1

# for altering a text column later
#column = [row[1] for row in value_list] # use a list comprehension to extract the column of interest
#max_length = max(len(element) for element in column) + 10 # find the maximum length of the elements in the column

print("Creating condition datasets for " + str(len(value_list)) + " ecological systems.")
print("===============================================================================")

for index in value_list:
    EVTvalue = index[0]
    EVTname = index[1]
    # create a clean version of the system name filenames
    pattern = re.compile('[^a-zA-Z0-9\s]+')
    EVTclean = re.sub(pattern, '', EVTname)
    pattern = re.compile('\s+')
    EVTclean = re.sub(pattern, '', EVTclean)

    print("working on " + EVTname + " (EVT " + str(EVTvalue) + ")")

    # Process: Extract by Attributes (Extract by Attributes) (sa)
    print("- working on extracting the EVT data")
    nameEVT = "tmp_Extract_" + str(EVTvalue) + ".tif"
    w_clause = '"VALUE" = ' + "%s" %EVTvalue
    tmp_Extract_Value_tif = ExtractByAttributes(in_raster="EVTclip.tif", where_clause=w_clause)
    tmp_Extract_Value_tif.save(os.path.join(arcpy.env.workspace, nameEVT))

    # Select out the hexagons that overlap the EVT cells
    print("- intersecting the EVT layer with the hex grid")
    tmp_Value_raster2pt = fr"tmp_{EVTvalue}_raster2pt"
    arcpy.conversion.RasterToPoint(in_raster=tmp_Extract_Value_tif, out_point_features=tmp_Value_raster2pt, raster_field="VALUE")
    hexgridselection = arcpy.management.SelectLayerByLocation(in_layer=[hexgrid], overlap_type="INTERSECT", select_features=tmp_Value_raster2pt, search_distance="", selection_type="NEW_SELECTION", invert_spatial_relationship="NOT_INVERT")
    hexgridselection = arcpy.conversion.FeatureClassToFeatureClass(in_features=hexgridselection, out_path=os.path.join(arcpy.env.workspace,HabitatCondition_gdb), out_name=f"hex100ac_{EVTclean}", where_clause="")

    arcpy.management.CalculateField(in_table=hexgridselection, field="EVTcode", expression=EVTvalue, field_type="LONG")
    arcpy.management.CalculateField(in_table=hexgridselection, field="EVTname", expression="'" + EVTname + "'", field_type="TEXT")
    #arcpy.management.AlterField(in_table=hexgridselection, field="EVTname", new_field_alias="EVT name", field_type="TEXT", field_length=max_length)

    # Work on the LCM
    print("- calculating and summarizing the LCM values")
    tmp_Value_LCM_tif = fr"tmp_{EVTvalue}_LCM.tif"
    tmp_Value_LCM_tif = arcpy.sa.ExtractByMask(in_raster=dataLCM, in_mask_data=nameEVT)
    ZonalSt_Value_LCM = fr"ZonalSt_{EVTvalue}_LCM"
    arcpy.sa.ZonalStatisticsAsTable(in_zone_data=hexgridselection, zone_field="GRID_ID", in_value_raster=tmp_Value_LCM_tif,
                                    out_table=ZonalSt_Value_LCM, ignore_nodata="DATA", statistics_type="MEAN")
    arcpy.management.JoinField(in_data=hexgridselection, in_field="GRID_ID", join_table=ZonalSt_Value_LCM, join_field="GRID_ID", fields=["MEAN"])[0]
    arcpy.management.CalculateField(in_table=hexgridselection, field="scoreLCM", expression="round(!MEAN!,1)", field_type="DOUBLE")
    arcpy.management.DeleteField(hexgridselection, "MEAN")
    arcpy.management.AlterField(hexgridselection, "scoreLCM", "scoreLCM", "LCM Score")

    # Work on Invasive Risk score
    print("- calculating the invasive risk score")
    tmp_InvRisk_tif = fr"tmp_{EVTvalue}_Inv.tif"
    tmp_InvRisk_tif = arcpy.sa.ExtractByMask(in_raster=dataRuderal, in_mask_data=nameEVT)
    ZonalSt_Value_InvRisk = fr"ZonalSt_{EVTvalue}_InvRisk"
    arcpy.sa.ZonalStatisticsAsTable(in_zone_data=hexgridselection, zone_field="GRID_ID", in_value_raster=tmp_InvRisk_tif,
                                    out_table=ZonalSt_Value_InvRisk, ignore_nodata="DATA", statistics_type="MEAN")
    arcpy.management.JoinField(in_data=hexgridselection, in_field="GRID_ID", join_table=ZonalSt_Value_InvRisk, join_field="GRID_ID", fields=["MEAN"])[0]
    arcpy.management.CalculateField(in_table=hexgridselection, field="scoreInv", expression="round(!MEAN!,1)", field_type="DOUBLE")
    arcpy.management.DeleteField(hexgridselection, "MEAN")
    arcpy.management.AlterField(hexgridselection, "scoreInv", "scoreInv", "Invasive Score")

    # calculating the mean condition/quality score
    arcpy.management.CalculateField(in_table=hexgridselection, field="scoreInvR", expression="Abs($feature.scoreInv-100)", expression_type="ARCADE", code_block="", field_type="DOUBLE", enforce_domains="NO_ENFORCE_DOMAINS")
    arcpy.management.CalculateField(in_table=hexgridselection, field="meanCond", expression="(!scoreLCM!+!scoreInvR!)/2", expression_type="PYTHON3", code_block="", field_type="DOUBLE", enforce_domains="NO_ENFORCE_DOMAINS")
    arcpy.management.AlterField(hexgridselection, "meanCond", "meanCond", "Mean Condition")

    # Work on fire departure score
    print("- calculating the fire departure score")
    tmp_FireDep_tif = fr"tmp_{EVTvalue}_Inv.tif"
    tmp_FireDep_tif = arcpy.sa.ExtractByMask(in_raster=dataFireDep, in_mask_data=nameEVT)
    ZonalSt_Value_FireDep = fr"ZonalSt_{EVTvalue}_InvRisk"
    arcpy.sa.ZonalStatisticsAsTable(in_zone_data=hexgridselection, zone_field="GRID_ID", in_value_raster=tmp_FireDep_tif,
                                    out_table=ZonalSt_Value_FireDep, ignore_nodata="DATA", statistics_type="MEAN")
    arcpy.management.JoinField(in_data=hexgridselection, in_field="GRID_ID", join_table=ZonalSt_Value_FireDep, join_field="GRID_ID", fields=["MEAN"])[0]
    arcpy.management.CalculateField(in_table=hexgridselection, field="scoreFire", expression="round(!MEAN!,1)", field_type="DOUBLE")
    arcpy.management.DeleteField(hexgridselection, "MEAN")
    arcpy.management.AlterField(hexgridselection, "scoreFire", "scoreFire", "Fire Departure Score")

    # apply a layer file
##    print("- creating the layer files")
##    dst = str(hexgridselection) + ".lyrx"
##    #shutil.copyfile(templateLCM, dst)
##    arcpy.management.ApplySymbologyFromLayer(hexgridselection, templateLCM, "VALUE_FIELD scoreLCM scoreLCM", "DEFAULT")
##    arcpy.management.SaveToLayerFile(hexgridselection, dst, "RELATIVE")

    # clean up
    print("- Cleaning up the crumbs")
    arcpy.management.Delete(in_data=[tmp_Value_raster2pt])

# higher level cleanup

#arcpy.management.Delete(scratchWorkspace)


import os
from tqdm import tqdm
import cv2
import random
import numpy as np
import pickle

categories = ["tulipan", "roza", "mniszek", "slonecznik", "stokrotka"]

datadir = './/Praca MGR//flower_photos//Train'
img_size = 50
training_data = []

def create_training_data():
    print("[STATUS] Przygotowanie danych uczących...")
    for category in categories:
        path = os.path.join(datadir,category)
        class_num = categories.index(category)
        print("[STATUS] ", class_num+1, "/ 5")
        print("Kategoria (etykieta) : ", category)
        print("Ścieżka do pliku     : ", path)
        print("Numer kategorii      : ", class_num)
        for img in tqdm(os.listdir(path)):
            #print(img)
            try:
                img_array = cv2.imread(os.path.join(path,img))  # convert to array
                new_array = cv2.resize(img_array, (img_size, img_size))  # resize to normalize data size
                training_data.append([new_array, class_num])  # add this to our training_data
            except Exception as e:  # in the interest in keeping the output clean...
                pass
        print(" ")

create_training_data()

create_training_data()
print("[STATUS] Znaleziono ", len(training_data), " obrazów. ")

random.shuffle(training_data)
#print("[STATUS] Przetasowano dane.")

X = []
y = []

for features,label in training_data:
    X.append(features)
    y.append(label)

X = np.array(X)

print(X.shape)
test = 1275*50*50*3
print(test)

print(X.shape)

pickle_out = open("X.pickle","wb")
pickle.dump(X, pickle_out)
pickle_out.close()

pickle_out = open("y.pickle","wb")
pickle.dump(y, pickle_out)
pickle_out.close()