print("Do You Want To Calculate Something Now ?")
print("(1) Yes")
print("(2) No")
yesOrNo=str(input("Enter Your Choice Here (1 or 2): "))
if yesOrNo=="1" :
    # ----------------calculator start-------------------
    print("what Operator You Want To Calculate")
    print("\t (1)Jog")
    print("\t (2)Biog")
    print("\t (3)Gun")
    print("\t (4)Vag")
    print("\t (5)Bagshes")
    choiceOperator = int(input("Enter Your Choice Here (1 or 2 or 3 or 4 or 5):"))

    # =================JOGFOL==================

    if choiceOperator==1:
      how_many_number = int(input("How Many Nummber You want to add : "))
      if how_many_number < 2:
          print("you need at least Two Number For Addition")
      else:
           a = 0
           for i in range(1,how_many_number+1):
               enternumber =  int(input(f"Enter Your {i} number :" ) )
               c = enternumber + a
               a = c
           print(a)

    # =================Biogfol==================

    elif choiceOperator==2:
        biognumber1=int(input("Enter Your First Number : "))
        biognumber2=int(input("Enter Your secound Number : "))
        biogfol = biognumber1 - biognumber2
        print(f"Biogfol :  {biogfol}")

    # =================GUNFOL==================

    elif choiceOperator==3:
       gunsongkha = int(input("How Many Number You Want For Multiplication : "))
       if gunsongkha < 2 :
           print("Enter atleast Two Number for Multipication")
       else:
           count=1;
           for i in range(1,gunsongkha+1):
                gunnumber = int(input(f"Enter Your {i} Number :") )
                Ga = gunnumber * count  ;
                count = Ga;
           print(f"Result : {Ga}")

    # =================BAGFOL==================

    elif choiceOperator==4:
        bagnumber1=int(input("Enter Your First Number : "))
        bagnumber2=int(input("Enter Your secound Number : "))
        bagfol = bagnumber1 / bagnumber2
        print(f"Bagfol :  {bagfol}")

    # =================BAGSHES==================

    elif choiceOperator==5:
        baghshesnumber1=int(input("Enter Your First Number : "))
        baghshesnumber2=int(input("Enter Your secound Number : "))
        if baghshesnumber1 % baghshesnumber2 == 0:
            print("No Bagshes")
            bagshesResult = baghshesnumber1/baghshesnumber2;
            print(f"Bagfol Result : {bagshesResult}")
        else:
            if  baghshesnumber1 > baghshesnumber2 :
             bagshesholebag=int(baghshesnumber1/baghshesnumber2)
             gunkorepai = int(bagshesholebag * baghshesnumber2  )
             bagshesbiiog = baghshesnumber1 - gunkorepai
             print(f"Bagshes: {bagshesbiiog}")
            else:
             print("Bajjo Bajok er thaka choto Hole Dosomik a man asbe")
             dosomiman=  float(baghshesnumber1 / baghshesnumber2 ) ;
             print(f"Dosomik Result : {dosomiman}")

        # ----------------calculator end-------------------

    else:
        print("please Inpur Corrcet Value from (1 or 2 or 3 or 4 or 5)")

elif yesOrNo=="2":
        print("Thank you For Visiting Us")
elif yesOrNo== "":
        print("Without Inpur Your Choice You Can Not Calculate Anything")
else:
    print("plese Input correct value")


"""Train CIFAR10 with PyTorch."""

import argparse
import logging
import os
import time

import torch.nn as nn
import torch.optim as optim
from torch.backends import cudnn
from torch.utils.tensorboard import SummaryWriter

from utils.deepfool import *
from models.preact_resnet import PreActResNet18
from models.resnet import ResNet18

def get_args():
    parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
    parser.add_argument('--dataset_path', default='./data', help='path of the dataset')

    parser.add_argument('--model', '-m', default='PreActResNet18', type=str)
    parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
    parser.add_argument('--lr_schedule', default='multistep', choices=['multistep', 'constant', 'cyclic'])
    parser.add_argument('--lr-min', default=0., type=float)
    parser.add_argument('--lr-max', default=0.3, type=float)
    parser.add_argument('--lr_change_epoch', nargs='+', default=[100, 150], type=int)
    parser.add_argument('--batch_size', '-b', default=256, type=int)
    parser.add_argument('--num_epochs', default=200, type=int)

    parser.add_argument('--epsilon', default=8, type=int)
    parser.add_argument('--train_fgsm_ratio', default=1, type=float)
    parser.add_argument('--eval_pgd_ratio', default=0.25, type=float)
    parser.add_argument('--eval_pgd_attack_iters', default=10, type=int)
    parser.add_argument('--eval_pgd_restarts', default=1, type=int)
    parser.add_argument('--grad_align_cos_lambda', default=2, type=float,
                        help='coefficient of the cosine gradient alignment regularizer')

    parser.add_argument('--finetune', action='store_true', help='finetune the pre-trained model with adversarial '
                                                                'samples or regularization')
    parser.add_argument('--resumed_model_name', default='standard_cifar.pth', help='the file name of resumed model')
    parser.add_argument('--seed', default=0, type=int)
    parser.add_argument('--save_epoch', action='store_true')
    parser.add_argument('--exp_name', default='standard_cifar', help='used as filename of saved model, '
                                                                     'tensorboard and log')
    return parser.parse_args()

def l2_norm_batch(v):
    norms = (v ** 2).sum([1, 2, 3]) ** 0.5
    return norms

def calculate_fgsm_delta(args, model, inputs, targets, normalize):
    zero = torch.zeros_like(inputs).to(args.device)
    zero.requires_grad = True
    output_input = model(normalize(inputs + zero))
    loss_input = F.cross_entropy(output_input, targets)
    grad_input = torch.autograd.grad(loss_input, zero, create_graph=True)[0]

    delta = torch.zeros_like(inputs).to(args.device)
    delta.uniform_(-args.epsilon, args.epsilon)
    delta.requires_grad = True
    output = model(normalize(inputs + delta))
    loss = F.cross_entropy(output, targets)
    grad_random = torch.autograd.grad(loss, delta, create_graph=True)[0]

    reg = torch.zeros(1).cuda()[0]  # for .item() to run correctly
    grads_nnz_idx = ((grad_input ** 2).sum([1, 2, 3]) ** 0.5 != 0) * ((grad_random ** 2).sum([1, 2, 3]) ** 0.5 != 0)
    grad1, grad2 = grad_input[grads_nnz_idx], grad_random[grads_nnz_idx]
    grad1_norms, grad2_norms = l2_norm_batch(grad1), l2_norm_batch(grad2)
    grad1_normalized = grad1 / grad1_norms[:, None, None, None]
    grad2_normalized = grad2 / grad2_norms[:, None, None, None]
    cos = torch.sum(grad1_normalized * grad2_normalized, (1, 2, 3))
    reg += args.grad_align_cos_lambda * (1.0 - cos.mean())

    fgsm_delta = args.train_fgsm_ratio * args.epsilon * torch.sign(grad_input)
    fgsm_delta = clamp(fgsm_delta, lower_limit - inputs, upper_limit - inputs).detach()
    fgsm_delta_norm = fgsm_delta.view(fgsm_delta.shape[0], -1).norm(dim=1)
    return fgsm_delta, fgsm_delta_norm, reg

# Training
def train(args, model, trainloader, normalize, optimizer, criterion, step_lr_scheduler):
    model.train()
    train_clean_loss, train_clean_correct, train_fgsm_loss, train_fgsm_correct, train_fgsm_delta_norm, train_total = 0, 0, 0, 0, 0, 0
    train_loss, train_reg = 0, 0
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        inputs, targets = inputs.to(args.device), targets.to(args.device)

        fgsm_delta, fgsm_delta_norm, reg = calculate_fgsm_delta(args, model, inputs, targets, normalize)
        fgsm_outputs = model(normalize(inputs + fgsm_delta))
        fgsm_loss = criterion(fgsm_outputs, targets)
        loss = fgsm_loss + reg

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        clean_outputs = model(normalize(inputs))
        clean_loss = criterion(clean_outputs, targets)

        train_loss += loss.item() * targets.size(0)
        train_reg += reg.item() * targets.size(0)
        train_fgsm_loss += fgsm_loss.item() * targets.size(0)
        train_fgsm_correct += (fgsm_outputs.max(dim=1)[1] == targets).sum().item()
        train_clean_loss += clean_loss.item() * targets.size(0)
        train_clean_correct += (clean_outputs.max(dim=1)[1] == targets).sum().item()
        train_fgsm_delta_norm += fgsm_delta_norm.sum().item()
        train_total += targets.size(0)
        if args.lr_schedule == 'cyclic':
            step_lr_scheduler.step()

    return train_clean_loss / train_total, 100. * train_clean_correct / train_total, train_fgsm_loss / train_total, 100. * train_fgsm_correct / train_total, train_fgsm_delta_norm / train_total, train_loss / train_total, train_reg / train_total

def eval(args, model, testloader, normalize, criterion, finaleval=False):
    model.eval()
    test_clean_loss, test_clean_correct, test_pgd10_loss, test_pgd10_correct, test_pgd10_delta_norm, test_total = 0, 0, 0, 0, 0, 0
    for batch_idx, (inputs, targets) in enumerate(testloader):
        inputs, targets = inputs.to(args.device), targets.to(args.device)

        # clean
        clean_outputs = model(normalize(inputs))
        clean_loss = F.cross_entropy(clean_outputs, targets)

        # pgd
        if finaleval:
            pgd_delta = attack_pgd(model, inputs, targets, normalize, args.epsilon, args.eval_pgd_ratio * args.epsilon,
                                   50, 10, args.device, early_stop=True).detach()
        else:
            pgd_delta = attack_pgd(model, inputs, targets, normalize, args.epsilon, args.eval_pgd_ratio * args.epsilon,
                                   args.eval_pgd_attack_iters, args.eval_pgd_restarts, args.device,
                                   early_stop=True).detach()
        pgd_outputs = model(normalize(inputs + pgd_delta))
        pgd_loss = criterion(pgd_outputs, targets)
        pgd_delta_norm = pgd_delta.view(pgd_delta.shape[0], -1).norm(dim=1)

        test_clean_loss += clean_loss.item() * targets.size(0)
        test_clean_correct += (clean_outputs.max(dim=1)[1] == targets).sum().item()
        test_pgd10_loss += pgd_loss.item() * targets.size(0)
        test_pgd10_correct += (pgd_outputs.max(dim=1)[1] == targets).sum().item()
        test_pgd10_delta_norm += pgd_delta_norm.sum().item()
        test_total += targets.size(0)

    return test_clean_loss / test_total, 100. * test_clean_correct / test_total, \
           test_pgd10_loss / test_total, 100. * test_pgd10_correct / test_total, test_pgd10_delta_norm / test_total

def tb_writer(writer, epoch, lr, train_clean_loss, train_clean_acc, train_fgsm_loss, train_fgsm_acc,
              train_fgsm_delta_norm, train_loss, train_reg,
              test_clean_loss, test_clean_acc, test_pgd10_loss, test_pgd10_acc, test_pgd10_delta_norm):
    writer.add_scalars('loss',
                       {'train_clean': train_clean_loss, 'train_fgsm': train_fgsm_loss, 'train_loss': train_loss,
                        'train_reg': train_reg,
                        'test_clean': test_clean_loss, 'test_pgd': test_pgd10_loss}, epoch)
    writer.add_scalars('accuracy',
                       {'train_clean': train_clean_acc, 'train_fgsm': train_fgsm_acc,
                        'test_clean': test_clean_acc, 'test_pgd': test_pgd10_acc}, epoch)
    writer.add_scalar('learning rate', lr, epoch)
    writer.add_scalars('delta_norm', {'train_fgsm': train_fgsm_delta_norm, 'test_pgd10': test_pgd10_delta_norm}, epoch)

def eval_init(args, writer, logger, model, trainloader, testloader, normalize, criterion, opt):
    model.eval()
    train_clean_loss, train_clean_correct, train_fgsm_loss, train_fgsm_correct, train_fgsm_delta_norm, train_total = 0, 0, 0, 0, 0, 0
    train_loss, train_reg = 0, 0
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        inputs, targets = inputs.to(args.device), targets.to(args.device)

        fgsm_delta, fgsm_delta_norm, reg = calculate_fgsm_delta(args, model, inputs, targets, normalize)
        fgsm_outputs = model(normalize(inputs + fgsm_delta))
        fgsm_loss = criterion(fgsm_outputs, targets)
        loss = fgsm_loss + reg

        clean_outputs = model(normalize(inputs))
        clean_loss = criterion(clean_outputs, targets)

        train_loss += loss.item() * targets.size(0)
        train_reg += reg.item() * targets.size(0)
        train_fgsm_loss += fgsm_loss.item() * targets.size(0)
        train_fgsm_correct += (fgsm_outputs.max(dim=1)[1] == targets).sum().item()
        train_clean_loss += clean_loss.item() * targets.size(0)
        train_clean_correct += (clean_outputs.max(dim=1)[1] == targets).sum().item()
        train_fgsm_delta_norm += fgsm_delta_norm.sum().item()
        train_total += targets.size(0)

    test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc, test_pgd10_delta_norm = eval(args, model, testloader,
                                                                                normalize, criterion)
    tb_writer(writer, 0, opt.param_groups[0]['lr'],
              train_clean_loss / train_total, 100. * train_clean_correct / train_total, train_fgsm_loss / train_total,
              100. * train_fgsm_correct / train_total, train_fgsm_delta_norm / train_total,
              train_loss / train_total, train_reg / train_total,
              test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc, test_pgd10_delta_norm)
    logger.info(
        '%d \t %.1f \t \t %.1f \t \t %.4f \t %.4f \t %.2f \t \t %.4f \t \t %.2f \t \t \t %.4f \t \t %.2f \t %.3f \t %.3f',
        0, -1, -1, opt.param_groups[0]['lr'], train_fgsm_loss / train_total, 100. * train_fgsm_correct / train_total,
        test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc, train_fgsm_delta_norm / train_total,
        test_pgd10_delta_norm)

def main():
    args = get_args()
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    OUTPUT_DIR = './output'
    LOG_DIR = './output/log'
    TENSORBOARD_DIR = './output/tensorboard'
    CHECKPOINT_DIR = './output/checkpoint/'
    if not os.path.exists(OUTPUT_DIR):
        os.mkdir(OUTPUT_DIR)
    if not os.path.exists(LOG_DIR):
        os.mkdir(LOG_DIR)
    if not os.path.exists(TENSORBOARD_DIR):
        os.mkdir(TENSORBOARD_DIR)
    if not os.path.exists(CHECKPOINT_DIR):
        os.mkdir(CHECKPOINT_DIR)

    writer = SummaryWriter(os.path.join(TENSORBOARD_DIR, args.exp_name))

    logger = logging.getLogger(__name__)
    logging.basicConfig(format='[%(asctime)s] - %(message)s',
                        datefmt='%Y/%m/%d %H:%M:%S',
                        level=logging.INFO,
                        filename=os.path.join(LOG_DIR, args.exp_name))

    # Model
    args.device = 'cpu'
    if torch.cuda.is_available():
        args.device = 'cuda'
        cudnn.benchmark = True
    logger.info(args)
    logger.info(f"model trained on {args.device}")

    trainloader, testloader = get_loader(args, logger)
    normalize = Normalize(cifar10_mu, cifar10_std)

    logger.info('==> Building model..')
    if args.model == 'ResNet18':
        model = ResNet18()
        logger.info("The model used is ResNet18")
    elif args.model == 'PreActResNet18':
        model = PreActResNet18()
        logger.info("The model used is PreActResNet18")
    else:
        logger.info("This model hasn't been defined ", args.model)
        raise NotImplementedError

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
    lr_steps = args.num_epochs * len(trainloader)
    if args.lr_schedule == 'cyclic':
        step_lr_scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=args.lr_min, max_lr=args.lr_max,
                                                              step_size_up=lr_steps / 2, step_size_down=lr_steps / 2)
    elif args.lr_schedule == 'multistep':
        step_lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=args.lr_change_epoch,
                                                           gamma=0.1)

    if torch.cuda.device_count() > 1:
        logger.info(f"Let's use {torch.cuda.device_count()} GPUs!")
        model = torch.nn.DataParallel(model)
    model = model.to(args.device)

    if args.finetune:
        # Load checkpoint.
        logger.info('==> Resuming from checkpoint..')
        assert os.path.isfile(os.path.join(CHECKPOINT_DIR,
                                           args.resumed_model_name)), f'Error: no asked checkpoint file {args.resumed_model_name} found! '
        checkpoint = torch.load(os.path.join(CHECKPOINT_DIR, args.resumed_model_name))
        model.load_state_dict(checkpoint['model'])
    args.epsilon = args.epsilon / 255.
    logger.info(
        'Epoch \t Train Time \t Test Time \t LR \t \t Train Loss \t Train Acc \t Test Standard Loss \t Test Standard '
        'Acc \t Test Attack Loss \t Test Attack Acc \t Train fgsm norm \t Test pgd norm')
    eval_init(args, writer, logger, model, trainloader, testloader, normalize, criterion, optimizer)
    best_test_pgd_acc = 0
    for epoch in range(args.num_epochs):
        start_time = time.time()
        train_clean_loss, train_clean_acc, train_fgsm_loss, train_fgsm_acc, train_fgsm_delta_norm, train_loss, train_reg = train(
            args, model, trainloader, normalize, optimizer, criterion, step_lr_scheduler)
        train_time = time.time()

        test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc, test_pgd10_delta_norm = eval(args, model,
                                                                                testloader,
                                                                                normalize, criterion)
        test_time = time.time()

        tb_writer(writer, epoch + 1, optimizer.param_groups[0]['lr'],
                  train_clean_loss, train_clean_acc, train_fgsm_loss, train_fgsm_acc, train_fgsm_delta_norm, train_loss,
                  train_reg,
                  test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc, test_pgd10_delta_norm)
        logger.info(
            '%d \t %.1f \t \t %.1f \t \t %.4f \t %.4f \t %.2f \t \t %.4f \t \t %.2f \t \t \t %.4f \t \t %.2f \t %.3f \t %.3f',
            epoch + 1, train_time - start_time, test_time - train_time, optimizer.param_groups[0]['lr'],
            train_fgsm_loss, train_fgsm_acc, test_clean_loss, test_clean_acc, test_pgd_loss, test_pgd_acc,
            train_fgsm_delta_norm, test_pgd10_delta_norm)
        if args.lr_schedule == 'multistep':
            step_lr_scheduler.step()
        if args.save_epoch and epoch % 1 == 0:
            save_checkpoint(model, epoch + 1, train_fgsm_loss, train_fgsm_acc, test_clean_loss, test_clean_acc,
                            test_pgd_loss, test_pgd_acc,
                            os.path.join(CHECKPOINT_DIR, args.exp_name + f'_{epoch + 1}.pth'))
        if test_pgd_acc >= best_test_pgd_acc:
            save_checkpoint(model, epoch + 1, train_fgsm_loss, train_fgsm_acc, test_clean_loss, test_clean_acc,
                            test_pgd_loss, test_pgd_acc, os.path.join(CHECKPOINT_DIR, args.exp_name + f'_best.pth'))
            best_test_pgd_acc = test_pgd_acc

    save_checkpoint(model, epoch + 1, train_fgsm_loss, train_fgsm_acc, test_clean_loss, test_clean_acc, test_pgd_loss,
                    test_pgd_acc, os.path.join(CHECKPOINT_DIR, args.exp_name + f'_final.pth'))

    # Evaluate best and final
    logger.info('best')
    checkpoint = torch.load(os.path.join(CHECKPOINT_DIR, args.exp_name + f'_best.pth'))
    model.load_state_dict(checkpoint['model'])
    best_clean_loss, best_clean_acc, best_pgd_loss, best_pgd_acc, best_pgd_delta_norm = eval(args, model, testloader,
                                                                                normalize, criterion,
                                                                                finaleval=True)
    logger.info('%d \t %.4f \t \t %.2f \t \t \t %.4f \t \t %.2f \t %.2f', checkpoint['epoch'], best_clean_loss,
                best_clean_acc, best_pgd_loss, best_pgd_acc, best_pgd_delta_norm)

    logger.info('final')
    checkpoint = torch.load(os.path.join(CHECKPOINT_DIR, args.exp_name + f'_final.pth'))
    model.load_state_dict(checkpoint['model'])
    final_clean_loss, final_clean_acc, final_pgd_loss, final_pgd_acc, final_pgd_delta_norm = eval(args, model,
                                                                                testloader, normalize,
                                                                                criterion,
                                                                                finaleval=True)
    logger.info('%d \t %.4f \t \t %.2f \t \t \t %.4f \t \t %.2f \t %.2f', checkpoint['epoch'], final_clean_loss,
                final_clean_acc, final_pgd_loss, final_pgd_acc, final_pgd_delta_norm)

    writer.close()

if __name__ == "__main__":
    main()


"""This file is a TMP_Footlocker spider

scrapy crawl tmp_footlocker -a extract=1 -a url="http://sneakerjobs.com/"

seed URLs:
    http://sneakerjobs.com/
"""

from urlparse import urljoin
from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.exceptions import CloseSpider
from re import compile
from brightcorp.processors import Prefix, Strip

class TMP_Footlocker(ATSSpider):

    name = 'tmp_footlocker'
    allowed_domain_bynetloc = True
    pagination_url_param = '?pagenumber='

    PAGINATION_REGEX = compile(r'\d+$')

    def parse(self, response):
        sel = Selector(response)
        # Start by getting the brand names (companies) and url params for their job search results page
        brand_urls  = sel.xpath("//li[@class='popmenu byGrp']/div/div/ul/li/a/@href").extract()
        brand_names = sel.xpath("//li[@class='popmenu byGrp']/div/div/ul/li/a/text()").extract()

        # No brands, no scrape
        if not brand_urls or not brand_names:
            raise CloseSpider("No brands available.")

        # Go through all brand pages
        for brand_index in xrange(len(brand_urls)):
            # Get the name and the url parameters for each brand
            brand_url  = brand_urls[brand_index].replace('//sneakerjobs.com', '')
            brand_name = brand_names[brand_index].replace('Jobs', '');
            self.log('BRAND NAME: ', brand_name);

            # Create the url with the response url and the brand url parameters
            url = urljoin(response.url, brand_url)

            # Create the request and save the company name, start url and set the current search result page to 1
            request = Request(url, callback=self.parse_page)
            request.meta['company']      = brand_name
            request.meta['start_url']    = response.url # We need this to create the job urls
            request.meta['current_page'] = 1
            yield request

    def parse_page(self, response):
        sel = Selector(response)

        # Get data for all jobs
        job_urls      = sel.xpath("//td[@class='jobTitle']/a/@href").extract()
        job_titles    = sel.xpath("//td[@class='jobTitle']/a/text()").extract()
        job_locations = sel.xpath("//td[@class='location']/text()").extract()

        # No jobs, return
        if not job_urls or not job_titles or not job_locations:
            return

        # Go through the job list. Save the company name, job title and location
        for job_index in xrange(len(job_urls)):
            # Get the job's data
            job_url      = job_urls[job_index]
            job_title    = job_titles[job_index]
            job_location = job_locations[job_index]

            # Create the url with the response url and the brand url parameters
            url = urljoin(response.meta.get('start_url'), job_url)

            request = Request(url, callback=self.parse_job_callback())
            # Pass the company name, job title and job location to the parse job method
            request.meta['company']  = response.meta.get('company')
            request.meta['title']    = job_title
            request.meta['location'] = job_location
            yield request

        # Now check for the last page number (pagination step)
        last_page = sel.xpath("//span[@class='numberedCurrent']/text()").extract()
        if last_page:
            # Increase the current page count by 1 to get the next page
            next_page = str(int(response.meta.get('current_page')) + 1)
            # Make sure the scraper hasn't reached the last page
            if last_page[0] > next_page:
                # Create the next page's url (http://sneakerjobs.com/careers/eastbay-jobs?pagenumber=next_page)
                pagination_string = self.pagination_url_param + next_page
                # Check if the response url already has the pagination url param
                if self.pagination_url_param in response.url:
                    index = response.url.find(self.pagination_url_param)
                    url = urljoin(response.url[:index], pagination_string)
                else:
                    url = urljoin(response.url, pagination_string)
                # Create a request back to the parse_page method
                request = Request(url, callback=self.parse_page)
                request.meta['company']      = response.meta.get('company')
                request.meta['start_url']    = response.url # We need this to create the job urls
                request.meta['current_page'] = next_page
                yield request

    def parse_job(self, response):
        sel = Selector(response)
        loader = BrightcorpItemLoader(selector=sel)

        # All the job attributes should be passed by the parse page method except for the description
        loader.add_value('url', response.url)
        loader.add_value('company', response.meta['company'])
        loader.add_value('title', response.meta['title'])
        loader.add_value('location', response.meta['location'])
        # Get the description
        loader.add_xpath('description', '//div[@id="jobDesc"]')
        loader.add_xpath('referencenumber', '//span[@itemprop="description"]/*[contains(text(), "Req ID")]/following-sibling::text()[1]',
                         Strip(), Prefix('footlocker-%s' % response.meta['company'].replace(' ', '-')))

        yield loader.load_item()


# Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
"""Settings for tests launch."""
# pylint: disable=too-few-public-methods

# size of the header in px
SIZE_HEADER = 90
SIZE_PANE_HEADER = 90
SIZE_FOOTER = 40

class Section(object):
  """Section names."""
  APP = "webapp"
  LOGGING = "logging"
  PYTEST = "pytest"

class Values(object):
  """Setting parameters for test launch."""
  WAIT_FOR_APP_SERVER = "wait_for_app_server"
  BASE_URL = "base_url"
  PORT = "port"
  LEVEL = "level"
  FORMAT = "format"
  RERUN_FAILED_TEST = "rerun_failed_test"


# -*- coding: utf-8 -*-

import pandas as pd
import plotly.graph_objects as go
import numpy as np
from scipy.stats import norm
import math
import plotly.graph_objects as go
import numpy as np
import kaleido
import os
import tweepy

ACCESS_TOKEN = os.getenv('ACCESS_TOKEN')
ACCESS_TOKEN_SECRET = os.getenv('ACCESS_TOKEN_SECRET')
API_KEY = os.getenv('API_KEY')
API_KEY_SECRET = os.getenv('API_KEY_SECRET')

def return_normal_distribution_candidate(sondage, erreur):
  data = np.arange(sondage-3*erreur, sondage+3*erreur, 0.1)
  x_out=[]
  for x in data:
    x_out += [norm(sondage, erreur).pdf(x)]
  return data, x_out

informations_candidats = {
    "Valérie Pécresse": {
        "couleur": "rgba(22, 64, 121, 255)"
    },
    "Eric Zemmour": {
      "couleur": "rgba(9, 16, 96, 255)"
    },
    "Emmanuel Macron": {
       "couleur": "rgba(89, 185, 249, 255)"
    },
    "Yannick Jadot": {
        "couleur": "rgba(122, 181, 27, 255)"
    },
    "Anne Hidalgo": {
        "couleur": "rgba(219, 65, 91, 255)"
    },
    "Marine Le Pen": {
        "couleur": "rgba(29, 55, 88, 255)"
    },
    "Jean-Luc Mélenchon": {
        "couleur": "rgba(150, 29, 31, 255)"
    },
}

def obtenir_couleur_candidat(candidat):
  return informations_candidats.get(candidat, {'couleur': 'black'})["couleur"]

def recuperer_donnees():
  id_sondages = pd.read_csv("id_sondages.csv")
  df = pd.read_csv("https://raw.githubusercontent.com/nsppolls/nsppolls/master/presidentielle.csv").sort_values(by="fin_enquete", ascending=False)
  df = df[df["tour"] == "Premier tour"]
  df = df.sort_values(by="intentions", ascending=False)
  return df, id_sondages

def exporter_tous_les_id_sondages(df):
  df.sort_values(by="id").id.drop_duplicates().to_csv("id_sondages.csv", index=False)

def graphique_violons(df):
  fig = go.Figure()
  candidats = df.candidat
  idx=4*(len(candidats))

  for candidat in candidats.tolist():
    sondage = df[df["candidat"] == candidat]["intentions"].values[0]
    erreur = df[df["candidat"] == candidat]["erreur_sup"].values[0] - sondage
    couleur = obtenir_couleur_candidat(candidat)

    if sondage>=2:
      x, y = return_normal_distribution_candidate(sondage, erreur)
      fig.add_trace(go.Scatter(x=x, y=[idx]*len(x), line_color=couleur, line_width=0, showlegend=False))
      fig.add_trace(go.Scatter(x=x, y=np.array(y)*10+idx, line_color=couleur, name=candidat, fill="tonexty", showlegend=False))
      fig.add_annotation(
          x=sondage,
          y=idx-0.8,
          text=candidat+"
"+str(round(sondage, 1))+" %",
          font=dict(color=couleur),
          showarrow=False
      )
      idx -= 4

  fig.update_xaxes(ticksuffix=" %", title="", tickfont=dict(size=15), side="top")
  fig.update_yaxes(visible=False)
  fig.update_layout(xaxis_showgrid=True, yaxis_showgrid=True, xaxis_zeroline=False)

  fig.add_annotation(
          x=0.5,
          y=1.13,
          xref="paper",
          yref="paper",
          text="Présidentielles : intentions de vote du dernier sondage",
          font=dict(size=25),
          showarrow=False
      )

  info_sondage = df["nom_institut"].values[0] + ", " + df["commanditaire"].values[0]
  dates = [df["debut_enquete"].values[0], df["fin_enquete"].values[0]]
  dates[0] = dates[0][-2:] + "/" + dates[0][-5:-3]
  dates[1] = dates[1][-2:] + "/" + dates[1][-5:-3]
  echantillon = df["echantillon"].values[0]

  fig.add_annotation(
          x=0.5,
          y=1.09,
          xref="paper",
          yref="paper",
          text=f"Sondage {info_sondage} du {dates[0]} au {dates[1]}, {echantillon} interrogés
@GuillaumeRozier • @ElecTracker_off • Données NSPPolls",
          font=dict(size=15),
          showarrow=False
      )

  fig.write_image(f"out/violin_{df.id.values[0]}.png", engine="kaleido", width=900, height=900, scale=2)

def tweet_sondage(df):
  info_sondage = df["nom_institut"].values[0] + ", " + df["commanditaire"].values[0]
  dates = [df["debut_enquete"].values[0], df["fin_enquete"].values[0]]
  candidats = df.candidat

  tweet = "Nouveau sondage " + info_sondage + "(" + dates[1] + ") :"

  for candidat in candidats[:6]:
    intentions = df[df["candidat"] == candidat]["intentions"].values[0]
    tweet += "\n" + candidat + " : " + str(intentions) + "%"
  print(tweet)

def main():
  df, id_sondages = recuperer_donnees()
  exporter_tous_les_id_sondages(df)
  df = df[~df["id"].isin(id_sondages["id"])]

  if len(df)>0:
    for id in df.id.unique():
      df_temp = df[df["id"]==id]
      graphique_violons(df_temp)
      #tweet_sondage(df_temp)

if __name__ == '__main__':
  main()


import pygame
from random import randint
import math
from sklearn.cluster import KMeans

def distance(p1,p2):
    return math.sqrt((p1[0]-p2[0])**2 + (p1[1]-p2[1])**2)
pygame.init()
pygame.display.set_caption("kmean visualization")

screen = pygame.display.set_mode((1200,700))
clock = pygame.time.Clock()
running = True

BACKGROUND = (124,124,124)
BLACK = (0,0,0)
BACKGROUND_PANEL = (249,255,230)
WHITE = (255,255,255)

RED = (255,0,0)
GREEN = (0,255,0)
BLUE = (0,0,255)
YELLOW = (147, 153, 35)
PURPLE = (255,0,255)
SKY = (0,255,255)
ORANGE = (255,125,25)
GRAPE = (100,25,125)
GRASS = (55,155,65)

COLORS = [RED,GREEN,BLUE,YELLOW,PURPLE,SKY,ORANGE,GRAPE,GRASS]

font = pygame.font.SysFont('sans',40,True)
font_small = pygame.font.SysFont('sans',20,True)
text_plus = font.render('+',True,WHITE)
text_minus = font.render('-',True,WHITE)
text_run = font.render('Run',True,WHITE)
text_random = font.render('Random',True,WHITE)
text_error = font.render('Error',True,WHITE)
text_algorithm = font.render('Algorithm',True,WHITE)
text_reset = font.render('Reset',True,WHITE)

K=0
error=0
points = []
cluster = []
labels = []
while running:
    clock.tick(60)
    screen.fill(BACKGROUND)

    mouse_x, mouse_y = pygame.mouse.get_pos()
    # Draw interface
    # Draw panel
    pygame.draw.rect(screen,BLACK,(50,50,700,500))
    pygame.draw.rect(screen,BACKGROUND_PANEL,(55,55,690,490))

    # K Button +
    pygame.draw.rect(screen,BLACK,(850,50,50,50))
    screen.blit(text_plus,(860,55))

    # K button -
    pygame.draw.rect(screen, BLACK, (950, 50, 50, 50))
    screen.blit(text_minus, (960, 55))

    # K value
    text_K = font.render('K = ' + str(K), True, BLACK)
    screen.blit(text_K, (1050, 55))

    # Run button
    pygame.draw.rect(screen,BLACK,(850,150,150,50))
    screen.blit(text_run, (860, 155))

    # Random button
    pygame.draw.rect(screen, BLACK, (850, 250, 150, 50))
    screen.blit(text_random, (860, 255))

    # Algorithm button
    pygame.draw.rect(screen, BLACK, (850, 450, 150, 50))
    screen.blit(text_algorithm, (860, 455))

    # Reset button
    pygame.draw.rect(screen, BLACK, (850, 550, 150, 50))
    screen.blit(text_reset, (860, 555))

    # draw mouse position when mouse is in panel
    if 50 < mouse_x < 750 and 50 < mouse_y < 550:
        text_point = font_small.render('('+str(mouse_x-50)+','+str(mouse_y-50)+')', True, BLACK)
        screen.blit(text_point, (mouse_x+5, mouse_y))
    # End draw interface

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        if event.type == pygame.MOUSEBUTTONDOWN:
            # mouse is in pannel
            if 50 < mouse_x < 750 and 50 < mouse_y < 550:
                labels = []
                point = [mouse_x-50,mouse_y-50]
                points.append(point)
            # button K +
            if 850 < mouse_x < 900 and 50 < mouse_y < 100:
                if K<9:
                    K+=1

            # button K -
            if 950 < mouse_x < 1000 and 50 < mouse_y < 100:
                if K>0:
                    K-=1

            # run button
            if 850 < mouse_x < 1000 and 150 < mouse_y < 200:
                # assign point to closest cluster
                labels = []
                if points == [] or cluster == []:
                    continue
                for p in points:
                    distances_to_cluster = []
                    for c in cluster:
                        dis = distance(p,c)
                        distances_to_cluster.append(dis)
                    min_dis= min(distances_to_cluster)
                    labels.append(distances_to_cluster.index(min_dis))

                # update cluster
                for i in range(K):
                    sum_x = 0
                    sum_y = 0
                    count = 0
                    for j in range(len(points)):
                        if labels[j] == i:
                            sum_x += points[j][0]
                            sum_y += points[j][1]
                            count += 1
                    if count != 0:
                        cluster[i] = [sum_x/count, sum_y/count]
                # print('run')

            # random button
            if 850 < mouse_x < 1000 and 250 < mouse_y < 300:
                cluster = []
                labels = []
                for i in range(K):
                    cluster_random = [randint(0,700),randint(0,500)]
                    cluster.append(cluster_random)

            # algorithm button
            if 850 < mouse_x < 1000 and 450 < mouse_y < 500:
                try:
                    model = KMeans(n_clusters=K).fit(points)
                    cluster = model.cluster_centers_
                    labels = model.predict(points)
                except:
                    print('error')
                # print('algorithm')

            # reset button
            if 850 < mouse_x < 1000 and 550 < mouse_y < 600:
                K = 0
                error = 0
                points = []
                cluster = []
                labels = []
                # print('reset')
    # Draw cluster
    for i in range(len(cluster)):
        pygame.draw.circle(screen, COLORS[i], (cluster[i][0] + 50, cluster[i][1] + 50), 6)

    # Draw points in panel
    for i in range(len(points)):
        pygame.draw.circle(screen,BLACK,(points[i][0]+50,points[i][1]+50),6)
        if labels == []:
            pygame.draw.circle(screen, WHITE, (points[i][0] + 50, points[i][1] + 50), 5)
        else:
            pygame.draw.circle(screen, COLORS[labels[i]], (points[i][0] + 50, points[i][1] + 50), 5)

    # error text
    if points != [] and labels != []:
        error = 0
        for i in range(len(points)):
            error += distance(points[i], cluster[labels[i]])
    text_error = font.render('Error = ' + str(int(error)), True, BLACK)
    screen.blit(text_error, (850, 350))
    pygame.display.flip()
pygame.quit()


# -*- coding:utf-8 -*-
# Author: Li Gaoyang<gylia@isoftstone.com>
from adminworkstation.models import SysPath

def insert_syspath(data):
    message = SysPath()
    num = len(SysPath.objects)
    new_syspath_id = num + 1
    message.syspath_id = new_syspath_id

    # 存储数据data[sysname, path_id]
    message.syspath_name = data[0]
    #dictdata = {data[1]:data[2]}
    #str_dictdata = str(dictdata)
    #message.syspath_list = str_dictdata

    # 保存
    message.save()


import pandas as pd
import PIL.Image as Image
from utils import imshow

path_to_data_root = '/home/harry/temp/projects/size-o/data/coins/'
csv = 'coin_locs.csv'

class Coins():

    def __init__(self, root=path_to_data_root, csv=csv, transform=None):
        self.root = root
        self.csv = pd.read_csv(root + csv)
        self.image_IDs = self.csv['ID'].tolist()
        self.transform = transform

    def __len__(self):
        return len(self.image_IDs)

    def __getitem__(self, idx):
        img_name = self.root + self.csv.iloc[idx, 0] + '.jpg'
        img = Image.open(img_name)

        bndbox = self.csv.iloc[idx, 1:].tolist()
        print(bndbox)
        return img, bndbox

coins = Coins()

for img, bndbox in coins:
    print(type(img))
    print(type(bndbox))
    imshow(img, bndbox)


from scipy.io import arff
from io import StringIO
import numpy as np
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import pandas as pd

#This loads the file containing the data from the UCI Machine Learning repository
data_file = '../data/ThoraricSurgery.arff'
data, meta = arff.loadarff(data_file)

outcomes_df = pd.DataFrame.from_records(data)
nominal_features = ['DGN', 'PRE6',  'PRE14']
TF_features = ['PRE7', 'PRE8', 'PRE9', 'PRE10', 'PRE11', 'PRE17', 'PRE19', 'PRE19',
              'PRE25', 'PRE30', 'PRE32', 'Risk1Yr']
numeric_features = ['PRE4', 'PRE5', 'AGE']
nominal_dict = {feature: [] for feature in nominal_features}
TF_dict = {feature: [] for feature in TF_features}

# This collects all of the categorical features and true/false features so that they can be preprocessed
for feature in nominal_features:
    nominal_dict[feature][:] = (pd.Series(outcomes_df[feature].values).unique().tolist())
for feature in TF_features:
    TF_dict[feature][:] = (pd.Series(outcomes_df[feature].values).unique().tolist())

#This for loop performs two of the preprocessing steps in our pipeline which are: one hot encoding
# and conversion of true/false strings to 0 and 1 respectively
for feature in outcomes_df.columns.values:
    new_column = []
    # This loop converts the categorical features into one hot encodings
    for i, row in outcomes_df.iterrows():
        if feature in nominal_features:
            encoding = np.zeros(len(nominal_dict[feature])).tolist()
            index = nominal_dict[feature].index(row[feature])
            encoding[index] = 1
            new_column.append(str(encoding))
        # If the feature is a true/false feature then it is converted to a binary value
        if feature in TF_features:
            outcomes_df.loc[i,feature] = TF_dict[feature].index(row[feature])
    if feature in nominal_features:
        outcomes_df.drop([feature], axis=1)
        outcomes_df[feature] = pd.DataFrame(new_column, columns=[feature])

# These functions sepsrate the dataset into the two classes so that the imbalance within the dataset
# can be observed and used for the following upsampling
Risk1Yr_1_df = outcomes_df.loc[outcomes_df['Risk1Yr'] == 1]
Risk1Yr_0_df = outcomes_df.loc[outcomes_df['Risk1Yr'] == 0]

# This for loop upsamples the patients who died within 1 year of their procedures since
# the dataset had a class imbalance and results in an approximate 1:1 ratio of the dataset
for i in range(4):
    outcomes_df = outcomes_df.append(Risk1Yr_1_df)

# Using Scikit-learn we split the dataset into an 80-20 training and test split that uses stratification to ensure
# that the proportion of classes in the dataset is maintained in both the training and test sets
outcomes_train, outcomes_test = train_test_split(outcomes_df, test_size=0.2, stratify=outcomes_df['Risk1Yr'].values.tolist())
outcomes_train_df = pd.DataFrame(outcomes_train)
outcomes_test_df = pd.DataFrame(outcomes_test)
print(outcomes_train_df.head(5))
# This function is used for standardizing the features to reduce the variance and improve the neural network
# performance
def standardization(x, mean, std):
    z_scores_np = (x - mean) / std
    return z_scores_np

# This function rescales the features using the standardization function for the training set
def train_standard_loop(keys):
    for key in keys:
        x_np = outcomes_train_df[key].values
        x_np_mean = x_np.mean()
        x_np_std = x_np.std()
        outcomes_train_df[key] = outcomes_train_df[key].apply(standardization,args=(x_np_mean,x_np_std))

# This function rescales the features using the standardization function for the test set
def test_standard_loop(keys):
    for key in keys[1:10]:
        x_np = outcomes_test_df[key].values
        x_np_mean = x_np.mean()
        x_np_std = x_np.std()
        outcomes_test_df[key] = outcomes_test_df[key].apply(standardization,args=(x_np_mean,x_np_std))

train_standard_loop(list(['PRE4', 'PRE5', 'AGE']))
test_standard_loop(list(['PRE4', 'PRE5', 'AGE']))

outcomes_train_df.to_csv(path_or_buf='outcomes_data_train.csv',index=False)
outcomes_test_df.to_csv(path_or_buf='outcomes_data_test.csv',index=False)


from flask import Flask, Blueprint, request, Response, jsonify, url_for,\
    redirect, render_template, abort, session

# import json

from db import ImageInfoRepo

image_info_bp = Blueprint('image_info_blueprint', __name__)

@image_info_bp.route('/')
def all_images():
    # abort(401)  # Unauthorized
    result = ImageInfoRepo.find_all_from_db(True)
    return jsonify(result)

    # resp = Response(json.dumps(result), mimetype='application/json')
    # resp.headers.add('Server', 'python flask')
    # return resp


from dataclasses import dataclass
from flask import current_app, g, request, Blueprint
from werkzeug.exceptions import Unauthorized

from .db import query_db

bp = Blueprint("user", __name__)

class Actor:
    """base abstract user type"""

    pass

@dataclass
class User(Actor):
    """Logged in user. Has an email address."""

    id: int
    email: str
    title: str
    location_id: int
    organization_id: int
    manager_id: int

    @classmethod
    def get(cls, id: int):
        record = query_db(
            "select id, email, title, location_id, organization_id, manager_id from users where email  = ?",
            [id],
            one=True,
        )
        if record:
            return cls(**record)
        else:
            raise Exception("user not found")

    @classmethod
    def lookup(cls, email: str):
        record = query_db(
            "select id, email, title, location_id, organization_id, manager_id from users where email  = ?",
            [email],
            one=True,
        )
        if record:
            return cls(**record)
        else:
            raise Exception("user not found")

class Guest(Actor):
    """Anonymous user."""

    def __str__(self):
        return "Guest"

@bp.before_app_request
def set_current_user():
    """Set the `current_user` from the authorization header (if present)"""
    if not "current_user" in g:
        email = request.headers.get("user")
        if email:
            try:
                g.current_user = User.lookup(email)
            except Exception as e:
                current_app.logger.exception(e)
                return Unauthorized("user not found")
        else:
            g.current_user = Guest()

@bp.route("/whoami")
def whoami():
    you = g.current_user
    if isinstance(you, User):
        organization = query_db(
            "select name from organizations where id = ?",
            [you.organization_id],
            one=True,
        )["name"]
        return f"You are {you.email}, the {you.title} at {organization}. (User ID: {you.id})"


import netCDF4
import numpy as np
import os

class File(object):
    def __init__(self, file, xdim, ydim, mode='a', format='NETCDF3_CLASSIC'):
        self.file = file
        if mode != 'w':
            assert os.path.exists(self.file), 'File not existing: ' + self.file
        self.F = netCDF4.Dataset(self.file, mode=mode, format=format)
        self.xdim = xdim
        self.ydim = ydim

    def close(self):
        self.F.close()

    def create_dimension(self, dimname, size):
        self.F.createDimension(dimname, size=size)

    def append_latitude(self, x, vname='latitude'):
        self.append_variable(vname, x)
        self.F.variables[vname].setncattr('long_name', 'latitude')
        self.F.variables[vname].setncattr('units', 'degrees_north')

    def append_longitude(self, x, vname='longitude'):
        self.append_variable(vname, x)
        self.F.variables[vname].setncattr('long_name', 'longitude')
        self.F.variables[vname].setncattr('units', 'degrees_east')

    def append_variable(self, vname, x):
        assert isinstance(x, np.ma.core.MaskedArray), 'Input needs to be masked array!'
        dummy = -99999.
        assert x.ndim == 2
        self._create_variable(vname, 'd', (self.ydim, self.xdim), fill_value=dummy)

        out = x.data
        out[x.mask] = dummy
        self.F.variables[vname][:,:] = out[:,:]
        try:
            # fval = self.F.variables[vname].getncattr('_FillValue')
            print('Fill value already existing ... will not be set!')
        except:
            self.F.variables[vname].setncattr('_FillValue', dummy)

    def _create_variable(self, vname, dtype, dim, complevel=6, zlib=True, fill_value=None):
        """
        varname : str
            name of variable
        dtype : str
            datatype of variable
        dim : tuple
            tuple specifying the dimensions of the variables
        zlib : bool
            compress outout using zlib
        complevel : int
            compression level
        fill_value : float
            fill value for data
        """
        if vname in self.F.variables.keys():
            print('The variable ' + vname + ' is already existing on the file! SKIPPING')
            return None

        if fill_value is not None:
            self.F.createVariable(vname, dtype, dimensions=dim,
                                  fill_value=fill_value, zlib=zlib, complevel=complevel)
        else:
            self.F.createVariable(
                vname, dtype, dimensions=dim, zlib=zlib, complevel=complevel)


import decodefunctions as Fcn
import random
import decodehex2
decoded=open('samplehex.csv','w')

def randombinary(l):
    binstr = ''
    for i in range(l):
        x = random.random()
        if x < .5:
            b = '0'
        else:
            b = '1'

        binstr = binstr + b
    return binstr

preamble='000000000000000000101111'

short_long='1'
user_location='1'
countrycode='0100111100'  #316 - Canada

for n in range(10000):

    ident=randombinary(49)
    pdf1=preamble+short_long+user_location+countrycode+ident
    bch1= Fcn.calcbch(('_'+pdf1+'0'*59), "1001101101100111100011", 25, 86, 107)
    newbin=pdf1+bch1
    supdata=randombinary(26)
    bch2 = Fcn.calcbch('_'+newbin+supdata+'0'*12, '1010100111001', 107, 133, 145)
    finalbin=newbin+supdata+bch2
    # print(Fcn.bin2hex(finalbin))
    hexcheck=Fcn.bin2hex(finalbin)
    if decodehex2.BeaconFGB(hexcheck).errors==[]:
        decoded.write(hexcheck+'\n')
decoded.close()


#=========================================================================
# 128 bits x 256 words SRAM model
#=========================================================================

from pymtl3                         import *
from pymtl3.passes.backends.verilog import TranslationConfigs
from .SramGenericPRTL               import SramGenericPRTL

class SRAM_128x256_1P( Component ):

  # Make sure widths match the .v

  def construct( s ):

    # clock (in PyMTL simulation it uses implict .clk port when
    # translated to Verilog, actual clock ports should be CE1

    s.CE1  = InPort ()           # clk
    s.WEB1 = InPort ()           # bar( write en )
    s.OEB1 = InPort ()           # bar( out en )
    s.CSB1 = InPort ()           # bar( whole SRAM en )
    s.A1   = InPort ( Bits8   )  # address
    s.I1   = InPort ( Bits128 )  # write data
    s.O1   = OutPort( Bits128 )  # read data

    # instantiate a generic sram inside

    s.sram_generic = SramGenericPRTL( 128, 256 )(
      CE1  = s.CE1,
      WEB1 = s.WEB1,
      OEB1 = s.OEB1,
      CSB1 = s.CSB1,
      A1   = s.A1,
      I1   = s.I1,
      O1   = s.O1,
    )

    s.config_verilog_translate = TranslationConfigs(
      translate             = False,
      no_synthesis          = True,
    )


"""
Multi criteria decision analysis
"""
from __future__ import division
from __future__ import print_function

import json
import os

import pandas as pd
import numpy as np
import cea.config
import cea.inputlocator
from cea.optimization.lca_calculations import LcaCalculations
from cea.technologies.heat_exchangers import calc_Cinv_HEX
import cea.optimization.distribution.network_optimization_features as network_opt
from cea.analysis.multicriteria.optimization_post_processing.locating_individuals_in_generation_script import \
    locating_individuals_in_generation_script, create_data_address_file
from math import ceil, log

__author__ = "Sreepathi Bhargava Krishna"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Shanshan Hsieh", "Sreepathi Bhargava Krishna"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def multi_criteria_main(locator, config):
    # local variables
    generation = config.multi_criteria.generations
    network_type = config.multi_criteria.network_type

    # This calculates the exact path of the individual
    # It might be that this inidividual was repeated already some generations back
    # so we make sure that a pointer to the right adress exists first
    if not os.path.exists(locator.get_address_of_individuals_of_a_generation(generation)):
        data_address = locating_individuals_in_generation_script(generation, locator)
    else:
        data_address = pd.read_csv(locator.get_address_of_individuals_of_a_generation(generation))

    # import data storing information about the geneartion
    generation_data = pd.read_csv(locator.get_optimization_individuals_in_generation(generation))

    # compile cost data
    if network_type == 'DC':
        # Preprocess data
        data_generation = preprocessing_generations_data(locator, generation, network_type)
        objectives = data_generation['population']
        individual_list = objectives.axes[0].values

        # This does the work
        compiled_data_df = pd.DataFrame()
        for i, individual in enumerate(individual_list):
            compiled_data_df = compiled_data_df.append(preprocessing_cost_data_DC(individual,
                                                                                locator,
                                                                                data_generation,
                                                                                data_address,
                                                                                config), ignore_index=True)
        compiled_data_df = compiled_data_df.assign(individual=individual_list)

    if network_type == 'DH':
        # Preprocess data
        data_generation = preprocessing_generations_data(locator, generation, config.multi_criteria.network_type)
        objectives = data_generation['population']
        individual_list = objectives.axes[0].values

        # compile the data
        compiled_data_df = pd.DataFrame()
        for i, individual in enumerate(individual_list):
            compiled_data_df = compiled_data_df.append(preprocessing_cost_data_DH(individual,
                                                                                locator,
                                                                                data_generation,
                                                                                data_address,
                                                                                config), ignore_index=True)
        compiled_data_df['individual'] = individual_list

    # normalize data
    compiled_data_df = normalize_compiled_data(compiled_data_df)
    # rank data
    compiled_data_df['TAC_rank'] = compiled_data_df['normalized_TAC'].rank(ascending=True)
    compiled_data_df['emissions_rank'] = compiled_data_df['normalized_emissions'].rank(ascending=True)
    compiled_data_df['prim_rank'] = compiled_data_df['normalized_prim'].rank(ascending=True)

    ## user defined mcda
    compiled_data_df['user_MCDA'] = compiled_data_df[
                                        'normalized_Capex_total'] * config.multi_criteria.capextotal * config.multi_criteria.economicsustainability + \
                                    compiled_data_df[
                                        'normalized_Opex'] * config.multi_criteria.opex * config.multi_criteria.economicsustainability + \
                                    compiled_data_df[
                                        'normalized_TAC'] * config.multi_criteria.annualizedcosts * config.multi_criteria.economicsustainability + \
                                    compiled_data_df[
                                        'normalized_emissions'] * config.multi_criteria.emissions * config.multi_criteria.environmentalsustainability + \
                                    compiled_data_df[
                                        'normalized_prim'] * config.multi_criteria.primaryenergy * config.multi_criteria.environmentalsustainability + \
                                    compiled_data_df[
                                        'normalized_renewable_share'] * config.multi_criteria.renewableshare * config.multi_criteria.socialsustainability

    compiled_data_df['user_MCDA_rank'] = compiled_data_df['user_MCDA'].rank(ascending=True)

    compiled_data_df.to_csv(locator.get_multi_criteria_analysis(generation))

    return

def normalize_compiled_data(compiled_data_df):
    # TAC
    if (max(compiled_data_df['TAC_Mio']) - min(compiled_data_df['TAC_Mio'])) > 1E-8:
        normalized_TAC = (compiled_data_df['TAC_Mio'] - min(compiled_data_df['TAC_Mio'])) / (
                max(compiled_data_df['TAC_Mio']) - min(compiled_data_df['TAC_Mio']))
    else:
        normalized_TAC = [1] * len(compiled_data_df['TAC_Mio'])
    # emission
    if (max(compiled_data_df['total_emissions_kiloton']) - min(compiled_data_df['total_emissions_kiloton'])) > 1E-8:
        normalized_emissions = (compiled_data_df['total_emissions_kiloton'] - min(
            compiled_data_df['total_emissions_kiloton'])) / (
                                       max(compiled_data_df['total_emissions_kiloton']) - min(
                                   compiled_data_df['total_emissions_kiloton']))
    else:
        normalized_emissions = [1] * len(compiled_data_df['total_emissions_kiloton'])
    # primary energy
    if (max(compiled_data_df['total_prim_energy_TJ']) - min(compiled_data_df['total_prim_energy_TJ'])) > 1E-8:
        normalized_prim = (compiled_data_df['total_prim_energy_TJ'] - min(compiled_data_df['total_prim_energy_TJ'])) / (
                max(compiled_data_df['total_prim_energy_TJ']) - min(compiled_data_df['total_prim_energy_TJ']))
    else:
        normalized_prim = [1] * len(compiled_data_df['total_prim_energy_TJ'])
    # capex
    if (max(compiled_data_df['Capex_total_Mio']) - min(compiled_data_df['Capex_total_Mio'])) > 1E-8:
        normalized_Capex_total = (compiled_data_df['Capex_total_Mio'] - min(compiled_data_df['Capex_total_Mio'])) / (
                max(compiled_data_df['Capex_total_Mio']) - min(compiled_data_df['Capex_total_Mio']))
    else:
        normalized_Capex_total = [1] * len(compiled_data_df['Capex_total_Mio'])
    # opex
    if (max(compiled_data_df['Opex_total_Mio']) - min(compiled_data_df['Opex_total_Mio'])) > 1E-8:
        normalized_Opex = (compiled_data_df['Opex_total_Mio'] - min(compiled_data_df['Opex_total_Mio'])) / (
                max(compiled_data_df['Opex_total_Mio']) - min(compiled_data_df['Opex_total_Mio']))
    else:
        normalized_Opex = [1] * len(compiled_data_df['Opex_total_Mio'])
    # renewable energy share
    if (max(compiled_data_df['renewable_share_electricity']) - min(
            compiled_data_df['renewable_share_electricity'])) > 1E-8:
        normalized_renewable_share = (compiled_data_df['renewable_share_electricity'] - min(
            compiled_data_df['renewable_share_electricity'])) / (
                                             max(compiled_data_df['renewable_share_electricity']) - min(
                                         compiled_data_df['renewable_share_electricity']))
    else:
        normalized_renewable_share = [1] * compiled_data_df['renewable_share_electricity']
    compiled_data_df = compiled_data_df.assign(normalized_TAC=normalized_TAC)
    compiled_data_df = compiled_data_df.assign(normalized_emissions=normalized_emissions)
    compiled_data_df = compiled_data_df.assign(normalized_prim=normalized_prim)
    compiled_data_df = compiled_data_df.assign(normalized_Capex_total=normalized_Capex_total)
    compiled_data_df = compiled_data_df.assign(normalized_Opex=normalized_Opex)
    compiled_data_df = compiled_data_df.assign(normalized_renewable_share=normalized_renewable_share)
    return compiled_data_df

def preprocessing_generations_data(locator, generation_number, district_netowrk_type):
    """
    compile data from all individuals in this generation
    :param locator:
    :param generation_number:
    :return:
    """

    # load data of geneartion
    with open(locator.get_optimization_checkpoint(generation_number), "rb") as fp:
        data = json.load(fp)

    # change units of population (e.g., form kW to MW)
    df_population, individual_barcode, individual_names = Change_units_population(generation_number, locator, data)

    # change units of the hall of fame (e.g., form kW to MW)
    df_halloffame = change_units_hall_of_fame(data)

    # get values of barcode of individuals
    df_individual_barcode = pd.DataFrame({'Name': individual_names,
                                          'individual_barcode': individual_barcode}).set_index("Name")

    # compile network barcodes
    df_network = df_network_barcodes(data, district_netowrk_type, individual_names)

    data_processed = {'population': df_population,
                      'halloffame': df_halloffame,
                      'network': df_network,
                      'spread': data['spread'],
                      'euclidean_distance': data['euclidean_distance'],
                      'individual_barcode': df_individual_barcode}

    print('compiled data of all individuals in this generation: ', generation_number)

    return data_processed

def df_network_barcodes(data, network_type, individual_names):
    network_dict = {}
    for i, ind in enumerate(individual_names):
        if network_type == 'DC':
            district_network_list = data['DCN_list_All']
        else:  # DH
            district_network_list = data['DHN_list_All']
        network_dict[ind] = str("".join(map(str, district_network_list)))
    df_network = pd.DataFrame.from_dict(network_dict, orient='index', columns=['network'])
    return df_network

def Change_units_population(generation_number, locator, data):
    # compile objectives of all individuals in this generation
    # convert to millions
    costs_Mio = [round(objectives[0] / 1000000, 2) for objectives in data['tested_population_fitness']]
    # convert to tons x 10^3 (kiloton)
    emissions_kiloton = [round(objectives[1] / 1000000, 2) for objectives in data['tested_population_fitness']]
    # convert to gigajoules x 10^3 (Terajoules)
    prim_energy_TJ = [round(objectives[2] / 1000000, 2) for objectives in data['tested_population_fitness']]
    individual_names = ['ind' + str(i) for i in range(len(costs_Mio))]
    df_population = pd.DataFrame({'Name': individual_names, 'costs_Mio': costs_Mio,
                                  'emissions_kiloton': emissions_kiloton,
                                  'prim_energy_TJ': prim_energy_TJ
                                  }).set_index("Name")
    # compile barcodes of all individuals in this generation
    individual_barcode = [[str(ind) if type(ind) == float else str(ind) for ind in
                           individual] for individual in data['tested_population']]
    return df_population, individual_barcode, individual_names

def change_units_hall_of_fame(data):
    # compile objectives of individuals in the hall of fame in this generation
    # convert to millions
    costs_Mio_HOF = [round(objectives[0] / 1000000, 2) for objectives in data['halloffame_fitness']]
    # convert to tons x 10^3
    emissions_kiloton_HOF = [round(objectives[1] / 1000000, 2) for objectives in data['halloffame_fitness']]
    # convert to gigajoules x 10^3
    prim_energy_TJ_HOF = [round(objectives[2] / 1000000, 2) for objectives in data['halloffame_fitness']]
    individual_names_HOF = ['ind' + str(i) for i in range(len(costs_Mio_HOF))]
    df_halloffame = pd.DataFrame({'Name': individual_names_HOF, 'costs_Mio': costs_Mio_HOF,
                                  'emissions_kiloton': emissions_kiloton_HOF,
                                  'prim_energy_TJ': prim_energy_TJ_HOF}).set_index("Name")
    return df_halloffame

def preprocessing_cost_data_DC(individual, locator, data_raw, data_address, config):
    # local
    string_network = data_raw['network'].loc[individual].values[0]
    total_demand = pd.read_csv(locator.get_total_demand())
    building_names = total_demand.Name.values
    data_address = data_address[data_address['individual_list'] == individual]

    # get data about individual (pointer)
    generation_number = data_address['generation_number_address'].values[0]
    individual_number = data_address['individual_number_address'].values[0]

    # get data about the activation patterns of these buildings (main units)
    data_costs = pd.read_csv(os.path.join(
        locator.get_optimization_slave_investment_cost_detailed_cooling(individual_number, generation_number)))
    data_cooling = pd.read_csv(
        os.path.join(locator.get_optimization_slave_cooling_activation_pattern(individual_number, generation_number)))
    data_electricity = pd.read_csv(os.path.join(
        locator.get_optimization_slave_electricity_activation_pattern_cooling(individual_number, generation_number)))
    data_emissions = pd.read_csv(
        os.path.join(locator.get_optimization_slave_investment_cost_detailed(individual_number, generation_number)))

    # Total CAPEX calculations
    # Absorption Chiller
    data_costs['Capex_total_ACH_USD'] = data_costs['Capex_ACH_USD'].values[0]
    data_costs['Opex_total_ACH_USD'] = np.sum(data_cooling['Opex_var_ACH_USD']) + data_costs['Opex_fixed_ACH_USD']

    # VCC
    data_costs['Capex_total_VCC_USD'] = data_costs['Capex_VCC_USD'].values[0]
    data_costs['Opex_total_VCC_USD'] = np.sum(data_cooling['Opex_var_VCC_USD']) + data_costs['Opex_fixed_VCC_USD']

    # VCC Backup
    data_costs['Capex_total_VCC_backup_USD'] = data_costs['Capex_VCC_backup_USD']
    data_costs['Opex_total_VCC_backup_USD'] = np.sum(data_cooling['Opex_var_VCC_backup_USD']) + data_costs[
        'Opex_fixed_VCC_backup_USD']

    # Storage Tank
    data_costs['Capex_total_storage_tank_USD'] = data_costs['Capex_Tank_USD']
    data_costs['Opex_total_storage_tank_USD'] = data_costs['Opex_fixed_Tank_USD']

    # Cooling Tower
    data_costs['Capex_total_CT_USD'] = data_costs['Capex_CT_USD']
    data_costs['Opex_total_CT_USD'] = np.sum(data_cooling['Opex_var_CT_USD']) + data_costs['Opex_fixed_CT_USD']

    # CCGT
    data_costs['Capex_total_CCGT_USD'] = data_costs['Capex_CCGT_USD']
    data_costs['Opex_total_CCGT_USD'] = np.sum(data_cooling['Opex_var_CCGT_USD']) + data_costs['Opex_fixed_CCGT_USD']

    # pump
    data_costs['Capex_total_pumps_USD'] = data_emissions['Capex_pump'].values[0]
    data_costs['Opex_total_pumps_USD'] = data_costs['Opex_fixed_pump_USD'] + data_costs['Opex_var_pump_USD']

    # Lake - No lake in singapore, should be modified in future
    data_costs['Opex_fixed_Lake_USD'] = [0]
    data_costs['Opex_total_Lake_USD'] = [0]
    data_costs['Capex_total_Lake_USD'] = [0]
    data_costs['Capex_a_Lake_USD'] = [0]

    # PV
    data_costs['Capex_total_PV_USD'] = data_emissions['Capex_PV'].values[0]
    data_costs['Opex_total_PV_USD'] = data_emissions['Opex_fixed_PV'].values[0]
    data_costs['Opex_fixed_PV_USD'] = data_emissions['Opex_fixed_PV'].values[0]
    data_costs['Capex_a_PV_USD'] = data_emissions['Capex_a_PV'].values[0]

    # Disconnected Buildings
    Capex_total_disconnected_USD = 0
    Opex_total_disconnected_USD = 0
    Capex_a_total_disconnected_USD = 0
    DCN_barcode = string_network  # TODO: switch between DCN and DHN

    for (index, building_name) in zip(DCN_barcode, building_names):
        if index is '0':
            df = pd.read_csv(locator.get_optimization_decentralized_folder_building_result_cooling(building_name,
                                                                                configuration='AHU_ARU_SCU'))
            dfBest = df[df["Best configuration"] == 1]

            if dfBest['VCC to AHU_ARU_SCU Share'].iloc[0] == 1:  # FIXME: Check for other options
                VCC_cost_data = pd.read_excel(locator.get_supply_systems(), sheet_name="Chiller")
                VCC_cost_data = VCC_cost_data[VCC_cost_data['code'] == 'CH3']
                max_VCC_chiller_size = max(VCC_cost_data['cap_max'].values)
                Inv_IR = (VCC_cost_data.iloc[0]['IR_%']) / 100
                Inv_LT = VCC_cost_data.iloc[0]['LT_yr']

            if dfBest['single effect ACH to AHU_ARU_SCU Share (FP)'].iloc[0] == 1:
                Absorption_chiller_cost_data = pd.read_excel(locator.get_supply_systems(),
                                                             sheet_name="Absorption_chiller")
                Absorption_chiller_cost_data = Absorption_chiller_cost_data[
                    ['type', 'code', 'cap_min', 'cap_max', 'a', 'b', 'c', 'd', 'e', 'IR_%', 'LT_yr', 'O&M_%']]
                Absorption_chiller_cost_data = Absorption_chiller_cost_data[
                    Absorption_chiller_cost_data['type'] == 'double']
                max_ACH_chiller_size = max(Absorption_chiller_cost_data['cap_max'].values)
                Inv_IR = (Absorption_chiller_cost_data.iloc[0]['IR_%']) / 100
                Inv_LT = Absorption_chiller_cost_data.iloc[0]['LT_yr']

            Opex_total_disconnected_USD += dfBest["Operation Costs [CHF]"].iloc[0]
            Capex_a_total_disconnected_USD += dfBest["Annualized Investment Costs [CHF]"].iloc[0]
            Capex_total_disconnected_USD += (
                    dfBest["Annualized Investment Costs [CHF]"].iloc[0] * ((1 + Inv_IR) ** Inv_LT - 1) / (
                Inv_IR) * (1 + Inv_IR) ** Inv_LT)
    data_costs['Capex_total_disconnected_Mio'] = Capex_total_disconnected_USD / 1000000
    data_costs['Opex_total_disconnected_Mio'] = Opex_total_disconnected_USD / 1000000
    data_costs['Capex_a_disconnected_Mio'] = Capex_a_total_disconnected_USD / 1000000

    data_costs['Capex_a_disconnected_USD'] = Capex_a_total_disconnected_USD
    data_costs['Opex_total_disconnected_USD'] = Opex_total_disconnected_USD

    data_costs['costs_Mio'] = data_raw['population']['costs_Mio'][individual]
    data_costs['emissions_kiloton'] = data_raw['population']['emissions_kiloton'][individual]
    data_costs['prim_energy_TJ'] = data_raw['population']['prim_energy_TJ'][individual]

    # Network costs
    network_features = network_opt.NetworkOptimizationFeatures(config,
                                                    locator)  # TODO: check if it is using the latest run of result, if yes, it is wrong
    network_costs_a_USD = network_features.pipesCosts_DCN_USD * DCN_barcode.count("1") / len(
        DCN_barcode)  # FIXME: what does this mean?
    data_costs['Network_costs_USD'] = network_costs_a_USD
    Inv_IR = 0.05
    Inv_LT = 20
    network_costs_total_USD = (network_costs_a_USD * ((1 + Inv_IR) ** Inv_LT - 1) / (Inv_IR) * (1 + Inv_IR) ** Inv_LT)
    data_costs['Network_costs_Total_USD'] = network_costs_total_USD
    # Substation costs
    substation_costs_a_USD = 0
    substation_costs_total_USD = 0
    for (index, building_name) in zip(DCN_barcode, building_names):
        if index == "1":
            if df_current_individual['Data Centre'][0] == 1:
                df = pd.read_csv(locator.get_optimization_substations_results_file(building_name),
                                 usecols=["Q_space_cooling_and_refrigeration_W"])
            else:
                df = pd.read_csv(locator.get_optimization_substations_results_file(building_name),
                                 usecols=["Q_space_cooling_data_center_and_refrigeration_W"])

            subsArray = np.array(df)

            Q_max_W = np.amax(subsArray)
            HEX_cost_data = pd.read_excel(locator.get_supply_systems(), sheet_name="HEX")
            HEX_cost_data = HEX_cost_data[HEX_cost_data['code'] == 'HEX1']
            # if the Q_design is below the lowest capacity available for the technology, then it is replaced by the least
            # capacity for the corresponding technology from the database
            if Q_max_W < HEX_cost_data.iloc[0]['cap_min']:
                Q_max_W = HEX_cost_data.iloc[0]['cap_min']
            HEX_cost_data = HEX_cost_data[
                (HEX_cost_data['cap_min'] <= Q_max_W) & (HEX_cost_data['cap_max'] > Q_max_W)]

            Inv_a = HEX_cost_data.iloc[0]['a']
            Inv_b = HEX_cost_data.iloc[0]['b']
            Inv_c = HEX_cost_data.iloc[0]['c']
            Inv_d = HEX_cost_data.iloc[0]['d']
            Inv_e = HEX_cost_data.iloc[0]['e']
            Inv_IR = (HEX_cost_data.iloc[0]['IR_%']) / 100
            Inv_LT = HEX_cost_data.iloc[0]['LT_yr']
            Inv_OM = HEX_cost_data.iloc[0]['O&M_%'] / 100

            InvC = Inv_a + Inv_b * (Q_max_W) ** Inv_c + (Inv_d + Inv_e * Q_max_W) * log(Q_max_W)

            Capex_a = InvC * (Inv_IR) * (1 + Inv_IR) ** Inv_LT / ((1 + Inv_IR) ** Inv_LT - 1)
            Opex_fixed = Capex_a * Inv_OM
            substation_costs_total_USD += InvC
            substation_costs_a_USD += Capex_a + Opex_fixed

    data_costs['Substation_costs_USD'] = substation_costs_a_USD
    data_costs['Substation_costs_Total_USD'] = substation_costs_total_USD
    # Electricity Details/Renewable Share
    lca = LcaCalculations(locator, config.detailed_electricity_pricing)

    data_costs['Total_electricity_demand_GW'] = (data_electricity['E_total_req_W'].sum()) / 1000000000  # GW
    data_costs['Electricity_for_hotwater_GW'] = (data_electricity['E_hotwater_total_W'].sum()) / 1000000000  # GW
    data_costs['Electricity_for_appliances_GW'] = (data_electricity['E_appliances_total_W'].sum()) / 1000000000  # GW

    renewable_share_electricity = (data_electricity['E_PV_directload_W'].sum() +
                                   data_electricity['E_PV_grid_W'].sum() + data_electricity[
                                       'E_PVT_directload_W'].sum() +
                                   data_electricity['E_PVT_grid_W'].sum()) * 100 / \
                                  (data_costs['Total_electricity_demand_GW'] * 1000000000)
    data_costs['renewable_share_electricity'] = renewable_share_electricity

    data_costs['Electricity_Costs_Mio'] = ((data_electricity['E_GRID_directload_W'].sum() +
                                            data_electricity[
                                                'E_total_to_grid_W_negative'].sum()) * lca.ELEC_PRICE.mean()) / 1000000

    data_costs['Capex_a_ACH_USD'] = data_costs['Capex_a_ACH_USD'].values[0]
    data_costs['Capex_a_VCC_USD'] = data_costs['Capex_a_VCC_USD'].values[0]
    data_costs['Capex_a_VCC_backup_USD'] = data_costs['Capex_a_VCC_backup_USD'].values[0]
    data_costs['Capex_a_CT_USD'] = data_costs['Capex_a_CT_USD'].values[0]
    data_costs['Capex_a_storage_tank_USD'] = data_costs['Capex_a_Tank_USD'].values[0]
    data_costs['Capex_a_total_pumps_USD'] = data_emissions['Capex_a_pump'].values[0]
    data_costs['Capex_a_CCGT_USD'] = data_costs['Capex_a_CCGT_USD'].values[0]
    data_costs['Capex_a_PV_USD'] = data_emissions['Capex_a_PV'].values[0]

    data_costs['Capex_a_total_Mio'] = (data_costs['Capex_a_ACH_USD'] + data_costs['Capex_a_VCC_USD'] + \
                                       data_costs['Capex_a_VCC_backup_USD'] + data_costs['Capex_a_CT_USD'] + data_costs[
                                           'Capex_a_storage_tank_USD'] + \
                                       data_costs['Capex_a_total_pumps_USD'] + data_costs['Capex_a_CCGT_USD'] +
                                       data_costs[
                                           'Capex_a_PV_USD'] + Capex_a_total_disconnected_USD + substation_costs_a_USD + network_costs_a_USD) / 1000000

    data_costs['Capex_total_Mio'] = (data_costs['Capex_total_ACH_USD'] + data_costs['Capex_total_VCC_USD'] + data_costs[
        'Capex_total_VCC_backup_USD'] + \
                                     data_costs['Capex_total_storage_tank_USD'] + data_costs['Capex_total_CT_USD'] +
                                     data_costs['Capex_total_CCGT_USD'] + \
                                     data_costs['Capex_total_pumps_USD'] + data_costs[
                                         'Capex_total_PV_USD'] + Capex_total_disconnected_USD + substation_costs_total_USD + network_costs_total_USD) / 1000000

    data_costs['Opex_total_Mio'] = (((data_costs['Opex_total_ACH_USD'] + data_costs['Opex_total_VCC_USD'] + data_costs[
        'Opex_total_VCC_backup_USD'] + \
                                      data_costs['Opex_total_storage_tank_USD'] + data_costs['Opex_total_CT_USD'] +
                                      data_costs['Opex_total_CCGT_USD'] + \
                                      data_costs['Opex_total_pumps_USD'] + Opex_total_disconnected_USD)) + data_costs[
                                        'Opex_total_PV_USD'] + \
                                    data_costs[
                                        'Total_electricity_demand_GW'] * 1000000000 * lca.ELEC_PRICE.mean()) / 1000000

    data_costs['TAC_Mio'] = data_costs['Capex_a_total_Mio'] + data_costs['Opex_total_Mio']

    # temporary fix for bug in emissions calculation, change it after executive course
    data_costs['total_emissions_kiloton'] = data_costs['emissions_kiloton'] - abs(
        2 * data_emissions['CO2_PV_disconnected'] / 1000000)
    data_costs['total_prim_energy_TJ'] = data_costs['prim_energy_TJ'] - abs(
        2 * data_emissions['Eprim_PV_disconnected'] / 1000000)

    return data_costs

def preprocessing_cost_data_DH(individual, locator, data_raw, data_address, config):
    # local variables
    string_network = data_raw['network'].loc[individual].values[0]
    total_demand = pd.read_csv(locator.get_total_demand())
    building_names = total_demand.Name.values
    individual_barcode_list = data_raw['individual_barcode'].loc[individual].values[0]

    # get generation/individual number
    data_address = data_address[data_address['individual_list'] == individual]
    generation_number = data_address['generation_number_address'].values[0]
    individual_number = data_address['individual_number_address'].values[0]

    # get data about the activation patterns of these buildings (main units)
    data_costs = pd.read_csv(
        os.path.join(locator.get_optimization_slave_investment_cost_detailed(individual_number, generation_number)))
    data_activation_pattern = pd.read_csv(
        os.path.join(locator.get_optimization_slave_heating_activation_pattern(individual_number, generation_number)))
    data_electricity = pd.read_csv(os.path.join(
        locator.get_optimization_slave_electricity_activation_pattern_heating(individual_number, generation_number)))
    data_emissions = pd.read_csv(
        os.path.join(locator.get_optimization_slave_investment_cost_detailed(individual_number, generation_number)))

    # Disconnected Buildings
    Capex_total_disconnected_USD = 0
    Opex_total_disconnected_USD = 0
    Capex_a_total_disconnected_USD = 0
    district_network_barcode = string_network

    for (index, building_name) in zip(district_network_barcode, building_names):
        if index is '0':
            df = pd.read_csv(locator.get_optimization_decentralized_folder_building_result_heating(building_name))
            dfBest = df[df["Best configuration"] == 1]
            Opex_total_disconnected_USD += dfBest["Operation Costs [CHF]"].iloc[0]
            Capex_a_total_disconnected_USD += dfBest["Annualized Investment Costs [CHF]"].iloc[0]
            Capex_total_disconnected_USD += dfBest["Total Costs [CHF]"].iloc[0]

    data_costs['Capex_total_disconnected_Mio'] = Capex_total_disconnected_USD / 1E6
    data_costs['Opex_total_disconnected_Mio'] = Opex_total_disconnected_USD / 1E6
    data_costs['Capex_a_disconnected_Mio'] = Capex_a_total_disconnected_USD / 1E6

    data_costs['Capex_a_disconnected_USD'] = Capex_a_total_disconnected_USD
    data_costs['Opex_total_disconnected_USD'] = Opex_total_disconnected_USD

    # get objectives
    data_costs['costs_Mio'] = data_raw['population']['costs_Mio'][individual]
    data_costs['emissions_kiloton'] = data_raw['population']['emissions_kiloton'][individual]
    data_costs['prim_energy_TJ'] = data_raw['population']['prim_energy_TJ'][individual]

    # get Network costs
    network_features = network_opt.NetworkOptimizationFeatures(config, locator)
    network_costs_a_USD = network_features.pipesCosts_DCN_USD * district_network_barcode.count("1") / len(
        district_network_barcode)
    data_costs['Network_costs_USD'] = network_costs_a_USD
    Inv_IR = 0.05
    Inv_LT = 20
    network_costs_total_USD = (network_costs_a_USD * ((1 + Inv_IR) ** Inv_LT - 1) / (Inv_IR) * (1 + Inv_IR) ** Inv_LT)
    data_costs['Network_costs_Total_USD'] = network_costs_total_USD
    # Substation costs
    substation_costs_a_USD = 0
    substation_costs_total_USD = 0
    for (index, building_name) in zip(district_network_barcode, building_names):
        if index == "1":
            df = pd.read_csv(locator.get_optimization_substations_results_file(building_name),
                             usecols=["Q_dhw_W", "Q_heating_W"])
            subsArray = np.array(df)

            Q_max_W = np.amax(subsArray[:, 0] + subsArray[:, 1])
            Capex_a_HEX_building_USD, Opex_fixed_HEX_building_USD, Capex_HEX_building_USD = calc_Cinv_HEX(Q_max_W,
                                                                                locator,
                                                                                config,
                                                                                'HEX1')
            substation_costs_total_USD += Capex_HEX_building_USD
            substation_costs_a_USD += Capex_a_HEX_building_USD + Opex_fixed_HEX_building_USD

    data_costs['Substation_costs_USD'] = substation_costs_a_USD
    data_costs['Substation_costs_Total_USD'] = substation_costs_total_USD

    # Electricity Details/Renewable Share
    lca = LcaCalculations(locator, config.detailed_electricity_pricing)

    data_costs['Total_electricity_demand_GW'] = (data_electricity['E_total_req_W'].sum()) / 1E9  # GW
    data_costs['Electricity_for_hotwater_GW'] = (data_electricity['E_hotwater_total_W'].sum()) / 1E9  # GW
    data_costs['Electricity_for_appliances_GW'] = (data_electricity['E_appliances_total_W'].sum()) / 1E9  # GW

    renewable_share_electricity = (data_electricity['E_PV_directload_W'].sum() +
                                   data_electricity['E_PV_grid_W'].sum() + data_electricity[
                                       'E_PVT_directload_W'].sum() +
                                   data_electricity['E_PVT_grid_W'].sum()) * 100 / \
                                  (data_costs['Total_electricity_demand_GW'] * 1E9)  # [%]
    data_costs['renewable_share_electricity'] = renewable_share_electricity

    data_costs['Electricity_Costs_Mio'] = ((data_electricity['E_GRID_directload_W'].sum() +
                                            data_electricity[
                                                'E_total_to_grid_W_negative'].sum()) * lca.ELEC_PRICE.mean()) / 1E6

    # analyzed criteria
    capex_a_sum = data_costs.filter(like='Capex_a').sum(axis=1)[0]
    data_costs['Capex_a_total_Mio'] = (
                                              capex_a_sum + Capex_a_total_disconnected_USD + substation_costs_a_USD + network_costs_a_USD) / 1E6
    capex_sum = data_costs.filter(like='Capex').sum(axis=1)[0] - capex_a_sum
    data_costs['Capex_total_Mio'] = (
                                            capex_sum + Capex_total_disconnected_USD + substation_costs_total_USD + network_costs_total_USD) / 1E6
    opex_sum = data_costs.filter(like='Opex').sum(axis=1)[0]
    data_costs['Opex_total_Mio'] = (opex_sum + data_costs[
        'Total_electricity_demand_GW'] * 1E9 * lca.ELEC_PRICE.mean()) / 1E6

    data_costs['TAC_Mio'] = data_costs['Capex_a_total_Mio'] + data_costs['Opex_total_Mio']

    # temporary fix for bug in emissions calculation, change it after executive course
    data_costs['total_emissions_kiloton'] = data_costs['emissions_kiloton'] - abs(
        2 * data_emissions['CO2_PV_disconnected'] / 1E6)
    data_costs['total_prim_energy_TJ'] = data_costs['prim_energy_TJ'] - abs(
        2 * data_emissions['Eprim_PV_disconnected'] / 1E6)

    return data_costs

def main(config):
    locator = cea.inputlocator.InputLocator(config.scenario)

    print("Running multicriteria with scenario = %s" % config.scenario)
    print("Running multicriteria for generation = %s" % config.multi_criteria.generations)

    multi_criteria_main(locator, config)

if __name__ == '__main__':
    main(cea.config.Configuration())


from expert import Expert
from sparse_gate import SparseGate

class Mixture():
    def __init__(self, x_train, y_train, x_test, y_test, experts, inputs, spark_context):
        self.gate = SparseGate(x_train, y_train, x_test, y_test, inputs, spark_context)
        self.experts = experts
        self.model_previous = None

    def load_expert_weights_and_set_trainable_layers(self,weights_file='../lib/weights/base_model_'):
        model = self.gate.gateModel
        for a in range(len(self.experts)):
            m = self.experts[a]
            file = weights_file + str(a) + '.h5'
            m.load_weights(file, by_name=True)
            for b in m.layers:
                for l in model.layers:
                    if (l.name == b.name):
                        l.set_weights(b.get_weights())
                        print("loaded layer "+str(l.name))

        for l in model.layers:
            if ('gate' in l.name or 'lambda' in l.name):
                l.trainable = True
                print("trainable gate layer: "+str(l.name))
            else:
                l.trainable = False

    def train_init(self, datagen, weights_file):
        for i in range(1, len(self.experts) + 1):
            print("---------------------------------------")
            print(str(i) + " of " + str(len(self.experts)))
            print("---------------------------------------")

            self.gate.gateModel = self.gate.create_gate_model(self.experts[:i])
            if i > 1:
                self.gate.load_gate_weights(self.model_previous)
            self.load_expert_weights_and_set_trainable_layers()
            self.gate.train_gate(datagen, weights_file)
            self.model_previous = self.gate.gateModel

    def add_expert(self, datagen, weights_file, expert):
        self.experts.append(expert)
        self.gate.load_gate_weights(self.gate.model, self.model_previous)
        self.load_expert_weights_and_set_trainable_layers()
        self.gate.train_gate(self.gate,weights_file)
        self.model_previous = self.gate.gateModel


#SPDX-License-Identifier: MIT
import pytest
import re
import logging
import sqlalchemy as s

from augur.application.db.session import DatabaseSession
from augur.application.config import AugurConfig
from augur.application.db.engine import get_database_string

logger = logging.getLogger(__name__)

default_repo_id = "25430"
default_repo_group_id = "10"

def create_full_routes(routes):
    full_routes = []
    for route in routes:
        route = re.sub("<default_repo_id>", default_repo_id, route)
        route = re.sub("<default_repo_group_id>", default_repo_group_id, route)
        route = "http://localhost:5000/api/unstable/" + route
        full_routes.append(route)
    return full_routes

@pytest.fixture
def test_db_engine():

    # creates database engine the normal way and then gets the database string
    db_string = get_database_string()

    db_string_without_db_name = re.search(r"(.+:\/\/.+\/).+", db_string).groups()[0]

    testing_db_string = db_string_without_db_name + "augur-test"

    yield s.create_engine(testing_db_string)

@pytest.fixture
def test_db_session(test_db_engine):
    session = DatabaseSession(logger, test_db_engine)

    yield session

    session.close()

@pytest.fixture
def test_db_config(test_db_session):
    return AugurConfig(logger, test_db_session)

# @pytest.fixture(scope="session")
# def augur_app():
#     augur_app = Application(disable_logs=True)
#     return augur_app

# @pytest.fixture(scope="session")
# def metrics(augur_app):
#     return augur_app.metrics

# @pytest.fixture(scope="session")
# def client(augur_app):
#     flask_client = initialize_components(augur_app, disable_housekeeper=True).load()
#     return flask_client.test_client()


import numpy as np
import sys
import time
from shapely.geometry import Polygon
import paddle
import unittest

from ext_op import matched_rbox_iou

def rbox2poly_single(rrect, get_best_begin_point=False):
    """
    rrect:[x_ctr,y_ctr,w,h,angle]
    to
    poly:[x0,y0,x1,y1,x2,y2,x3,y3]
    """
    x_ctr, y_ctr, width, height, angle = rrect[:5]
    tl_x, tl_y, br_x, br_y = -width / 2, -height / 2, width / 2, height / 2
    # rect 2x4
    rect = np.array([[tl_x, br_x, br_x, tl_x], [tl_y, tl_y, br_y, br_y]])
    R = np.array([[np.cos(angle), -np.sin(angle)],
                  [np.sin(angle), np.cos(angle)]])
    # poly
    poly = R.dot(rect)
    x0, x1, x2, x3 = poly[0, :4] + x_ctr
    y0, y1, y2, y3 = poly[1, :4] + y_ctr
    poly = np.array([x0, y0, x1, y1, x2, y2, x3, y3], dtype=np.float64)
    return poly

def intersection(g, p):
    """
    Intersection.
    """

    g = g[:8].reshape((4, 2))
    p = p[:8].reshape((4, 2))

    a = g
    b = p

    use_filter = True
    if use_filter:
        # step1:
        inter_x1 = np.maximum(np.min(a[:, 0]), np.min(b[:, 0]))
        inter_x2 = np.minimum(np.max(a[:, 0]), np.max(b[:, 0]))
        inter_y1 = np.maximum(np.min(a[:, 1]), np.min(b[:, 1]))
        inter_y2 = np.minimum(np.max(a[:, 1]), np.max(b[:, 1]))
        if inter_x1 >= inter_x2 or inter_y1 >= inter_y2:
            return 0.
        x1 = np.minimum(np.min(a[:, 0]), np.min(b[:, 0]))
        x2 = np.maximum(np.max(a[:, 0]), np.max(b[:, 0]))
        y1 = np.minimum(np.min(a[:, 1]), np.min(b[:, 1]))
        y2 = np.maximum(np.max(a[:, 1]), np.max(b[:, 1]))
        if x1 >= x2 or y1 >= y2 or (x2 - x1) < 2 or (y2 - y1) < 2:
            return 0.

    g = Polygon(g)
    p = Polygon(p)
    if not g.is_valid or not p.is_valid:
        return 0

    inter = Polygon(g).intersection(Polygon(p)).area
    union = g.area + p.area - inter
    if union == 0:
        return 0
    else:
        return inter / union

def matched_rbox_overlaps(anchors, gt_bboxes, use_cv2=False):
    """

    Args:
        anchors: [M, 5]  x1,y1,x2,y2,angle
        gt_bboxes: [M, 5]  x1,y1,x2,y2,angle

    Returns:
        macthed_iou: [M]
    """
    assert anchors.shape[1] == 5
    assert gt_bboxes.shape[1] == 5

    gt_bboxes_ploy = [rbox2poly_single(e) for e in gt_bboxes]
    anchors_ploy = [rbox2poly_single(e) for e in anchors]

    num = len(anchors_ploy)
    iou = np.zeros((num, ), dtype=np.float64)

    start_time = time.time()
    for i in range(num):
        try:
            iou[i] = intersection(gt_bboxes_ploy[i], anchors_ploy[i])
        except Exception as e:
            print('cur gt_bboxes_ploy[i]', gt_bboxes_ploy[i],
                  'anchors_ploy[j]', anchors_ploy[i], e)
    return iou

def gen_sample(n):
    rbox = np.random.rand(n, 5)
    rbox[:, 0:4] = rbox[:, 0:4] * 0.45 + 0.001
    rbox[:, 4] = rbox[:, 4] - 0.5
    return rbox

class MatchedRBoxIoUTest(unittest.TestCase):
    def setUp(self):
        self.initTestCase()
        self.rbox1 = gen_sample(self.n)
        self.rbox2 = gen_sample(self.n)

    def initTestCase(self):
        self.n = 1000

    def assertAllClose(self, x, y, msg, atol=5e-1, rtol=1e-2):
        self.assertTrue(np.allclose(x, y, atol=atol, rtol=rtol), msg=msg)

    def get_places(self):
        places = [paddle.CPUPlace()]
        if paddle.device.is_compiled_with_cuda():
            places.append(paddle.CUDAPlace(0))

        return places

    def check_output(self, place):
        paddle.disable_static()
        pd_rbox1 = paddle.to_tensor(self.rbox1, place=place)
        pd_rbox2 = paddle.to_tensor(self.rbox2, place=place)
        actual_t = matched_rbox_iou(pd_rbox1, pd_rbox2).numpy()
        poly_rbox1 = self.rbox1
        poly_rbox2 = self.rbox2
        poly_rbox1[:, 0:4] = self.rbox1[:, 0:4] * 1024
        poly_rbox2[:, 0:4] = self.rbox2[:, 0:4] * 1024
        expect_t = matched_rbox_overlaps(poly_rbox1, poly_rbox2, use_cv2=False)
        self.assertAllClose(
            actual_t,
            expect_t,
            msg="rbox_iou has diff at {} \nExpect {}\nBut got {}".format(
                str(place), str(expect_t), str(actual_t)))

    def test_output(self):
        places = self.get_places()
        for place in places:
            self.check_output(place)

if __name__ == "__main__":
    unittest.main()


import itertools

weapons = [
    ("Dagger", 8, 4, 0),
    ("Shortsword", 10, 5, 0),
    ("Warhammer", 25, 6, 0),
    ("Longsword", 40, 7, 0),
    ("Greataxe", 74, 8, 0),
]

armor = [
    ("None", 0, 0, 0),
    ("Leather", 13, 0, 1),
    ("Chainmail", 31, 0, 2),
    ("Splintmail", 53, 0, 3),
    ("Bandedmail", 75, 0, 4),
    ("Platemail", 102, 0, 5),
]

rings = [
    ("None", 0, 0, 0),
    ("None", 0, 0, 0),
    ("Damage +1", 25, 1, 0),
    ("Damage +2", 50, 2, 0),
    ("Damage +3", 100, 3, 0),
    ("Defense +1", 20, 0, 1),
    ("Defense +2", 40, 0, 2),
    ("Defense +3", 80, 0, 3),
]

bhp = 100
barmor = 2
bdamage = 8

php = 100

def simulate(pdamage, parmor):
    h = php
    i = bhp

    while True:
        i -= max(1, pdamage - barmor)
        if i <= 0:
            return True
        h -= max(1, bdamage - parmor)
        if h <= 0:
            return False

best_win = 10**100
worst_lose = 0

for w in weapons:
    for a in armor:
        for r, s in itertools.combinations(rings, 2):
            cost = w[1] + a[1] + r[1] + s[1]
            pdam = w[2] + a[2] + r[2] + s[2]
            parm = w[3] + a[3] + r[3] + s[3]
            if simulate(pdam, parm):
                best_win = min(best_win, cost)
            else:
                worst_lose = max(worst_lose, cost)

print(best_win)
print(worst_lose)


import random
from collections import namedtuple, deque
import numpy as np

class ReplayBuffer:
    """Fixed-size buffer to store experience tuples."""

    def __init__(self, buffer_size, batch_size):
        """Initialize a ReplayBuffer object.
        Params
        ======
            buffer_size: maximum size of buffer
            batch_size: size of each training batch
        """
        self.memory = deque(maxlen=buffer_size)  # internal memory (deque)
        self.batch_size = batch_size
        self.experience = namedtuple("Experience",
                                     field_names=["state", "action", "reward", "next_state", "done", "prev_reward"])

    def add(self, state, action, reward, next_state, done, prev_reward):
        """Add a new experience to memory."""
        e = self.experience(state, action, reward, next_state, done, prev_reward)
        self.memory.append(e)

    def sample(self):
        """Randomly sample a batch of experiences from memory."""
        if len(self.memory) > self.batch_size:
            return random.sample(self.memory, k=self.batch_size)
        else:
            return self.memory

    def sample_all(self):
        return self.memory

    def __len__(self):
        """Return the current size of internal memory."""
        return len(self.memory)

# code adapted from https://gist.github.com/monkut/2e8bec49b0659d941abc
class RunningVariance(object):
    def __init__(self, shape=(), epsilon=1e-3):
        self.epsilon = epsilon
        self.count = 0
        self.mean = np.zeros(shape)
        self.last_mean = np.zeros(shape)
        self.sq_distance = np.zeros(shape)  # square distance from the mean
        self.last_sq_distance = np.zeros(shape)
        self.stddev = np.ones(shape)

    def update(self, value):
        val = np.array(value)
        self.count += 1
        self.mean = self.last_mean + (val - self.last_mean) / self.count
        self.sq_distance = self.last_sq_distance + (val - self.last_mean) * (val - self.mean)
        if self.count > 1:
            self.stddev = np.sqrt(self.sq_distance / (self.count - 1))
            self.stddev[self.stddev == 0] = self.epsilon
        self.last_mean = self.mean
        self.last_sq_distance = self.sq_distance

    def normalize(self, value):
        val = np.array(value)
        return (val - self.mean) / self.stddev

    def denormalize(self, value):
        val = np.array(value)
        return val * self.stddev + self.mean


'''If we list all the natural numbers below 10 that are multiples of 3 or 5, we get 3, 5, 6 and 9. The sum of these multiples is 23.

Finish the solution so that it returns the sum of all the multiples of 3 or 5 below the number passed in.

Note: If the number is a multiple of both 3 and 5, only count it once. Also, if a number is negative, return 0(for languages that do have them)

Courtesy of projecteuler.net

'''
def solution(number):
    multiples = 0
    if number >= 0:
        for nums in range(1, number):
            if ((nums % 3 == 0) or (nums % 5 == 0)):
                if nums % 3 == 0 and nums % 5 == 0:
                    multiples += nums
                    continue
                else:
                    multiples += nums

        return multiples
    else:
        return 0

'''def solution(number):
    return sum(x for x in range(number) if x % 3 == 0 or x % 5 == 0)

print(solution(15))'''


# -*- coding: utf-8 -*-
import unittest

from itineraries import *
from weekday import *
from daytime import *
from coordinates import *
from itinerary_repath import *
from itinerary_stats import *
from itinerary_annealing import *

class AnnealingTestCase(unittest.TestCase):

    def _getTestItinerary(self, wp1loc):
        dt1 = DayTime(hour=6, minute=0)
        dt2 = DayTime(hour=6, minute=30)
        dt3 = DayTime(hour=7, minute=0)
        wp1 = RestrictedWaypoint(location=Location.parse(wp1loc), daytime=dt1, tolerance=Interval(minutes=15))
        wp2 = Waypoint(location=Location.parse("32, 34"), daytime=dt2)
        wp3 = RestrictedWaypoint(location=Location.parse("31, 33"), daytime=dt3, tolerance=Interval(minutes=15), users=[3])
        itinerary = Itinerary(places=3)
        itinerary.addWeekday(Monday)
        itinerary.addWeekday(Thursday)
        itinerary.addWaypoint(wp1)
        itinerary.addWaypoint(wp2)
        itinerary.addWaypoint(wp3)
        return itinerary

    # testing functions

    def test_simple_repath(self):
        statsStrategy = ItineraryStatsMeter.default()()
        repathStrategy = ItineraryRepathingStrategy.default()(
            statsStrategy = statsStrategy)
        annealingStrategy = SimpleItineraryAnnealingStrategy(
            statsStrategy = statsStrategy,
            repathingStrategy = repathStrategy)

        # Should be perfectly annealable b->a
        a = self._getTestItinerary("32, 33")
        b = self._getTestItinerary("32, 33.2")

        self.assertTrue(annealingStrategy.canAnneal(b,a))

        # should not blow up
        self.assertGreater(len(annealingStrategy.anneal(b,a,"pepito").waypoints), 1)

if __name__ == '__main__':
    unittest.main()