from __future__ import print_function

import os
import os.path
import numpy as np
import random
import pickle
import json
import math

import torch
import torch.utils.data as data
import torchvision
import torchvision.datasets as datasets
import torchvision.transforms as transforms
import torchnet as tnt

import h5py

from PIL import Image
from PIL import ImageEnhance

from pdb import set_trace as breakpoint

# Set the appropriate paths of the datasets here.
_MINI_IMAGENET_DATASET_DIR = './datasets/MiniImagenet'
_IMAGENET_DATASET_DIR = './datasets/IMAGENET/ILSVRC2012'

_IMAGENET_LOWSHOT_BENCHMARK_CATEGORY_SPLITS_PATH = './data/IMAGENET_LOWSHOT_BENCHMARK_CATEGORY_SPLITS.json'

transformtypedict=dict(
    Brightness=ImageEnhance.Brightness, Contrast=ImageEnhance.Contrast,
    Sharpness=ImageEnhance.Sharpness, Color=ImageEnhance.Color
)

def get_label_ids(class_to_idx, class_names, inside=True):
    label_ids = []
    if inside:
        for cname in class_names:
            label_ids.append(class_to_idx[cname])
    else:
        for cname, clabel in class_to_idx.items():
            if cname not in class_names:
                label_ids.append(clabel)

    return label_ids

class ImageJitter(object):
    def __init__(self, transformdict):
        self.transforms = [(transformtypedict[k], transformdict[k]) for k in transformdict]

    def __call__(self, img):
        out = img
        randtensor = torch.rand(len(self.transforms))

        for i, (transformer, alpha) in enumerate(self.transforms):
            r = alpha*(randtensor[i]*2.0 -1.0) + 1
            out = transformer(out).enhance(r).convert('RGB')

        return out

class Denormalize(object):
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, tensor):
        for t, m, s in zip(tensor, self.mean, self.std):
            t.mul_(s).add_(m)
        return tensor

def buildLabelIndex(labels):
    label2inds = {}
    for idx, label in enumerate(labels):
        if label not in label2inds:
            label2inds[label] = []
        label2inds[label].append(idx)

    return label2inds

def load_data(file):
    with open(file, 'rb') as fo:
        data = pickle.load(fo)
    return data

class MiniImageNet(data.Dataset):
    def __init__(self, phase='train', do_not_use_random_transf=False):

        self.base_folder = 'miniImagenet'
        assert(phase=='train' or phase=='val' or phase=='test')
        self.phase = phase
        self.name = 'MiniImageNet_' + phase

        print('Loading mini ImageNet dataset - phase {0}'.format(phase))
        file_train_categories_train_phase = os.path.join(
            _MINI_IMAGENET_DATASET_DIR,
            'miniImageNet_category_split_train_phase_train.pickle')
        file_train_categories_val_phase = os.path.join(
            _MINI_IMAGENET_DATASET_DIR,
            'miniImageNet_category_split_train_phase_val.pickle')
        file_train_categories_test_phase = os.path.join(
            _MINI_IMAGENET_DATASET_DIR,
            'miniImageNet_category_split_train_phase_test.pickle')
        file_val_categories_val_phase = os.path.join(
            _MINI_IMAGENET_DATASET_DIR,
            'miniImageNet_category_split_val.pickle')
        file_test_categories_test_phase = os.path.join(
            _MINI_IMAGENET_DATASET_DIR,
            'miniImageNet_category_split_test.pickle')

        if self.phase=='train':
            # During training phase we only load the training phase images
            # of the training categories (aka base categories).
            data_train = load_data(file_train_categories_train_phase)
            self.data = data_train['data']
            self.labels = data_train['labels']

            self.label2ind = buildLabelIndex(self.labels)
            self.labelIds = sorted(self.label2ind.keys())
            self.num_cats = len(self.labelIds)
            self.labelIds_base = self.labelIds
            self.num_cats_base = len(self.labelIds_base)

        elif self.phase=='val' or self.phase=='test':
            if self.phase=='test':
                # load data that will be used for evaluating the recognition
                # accuracy of the base categories.
                data_base = load_data(file_train_categories_test_phase)
                # load data that will be use for evaluating the few-shot recogniton
                # accuracy on the novel categories.
                data_novel = load_data(file_test_categories_test_phase)
            else: # phase=='val'
                # load data that will be used for evaluating the recognition
                # accuracy of the base categories.
                data_base = load_data(file_train_categories_val_phase)
                # load data that will be use for evaluating the few-shot recogniton
                # accuracy on the novel categories.
                data_novel = load_data(file_val_categories_val_phase)

            self.data = np.concatenate(
                [data_base['data'], data_novel['data']], axis=0)
            self.labels = data_base['labels'] + data_novel['labels']

            self.label2ind = buildLabelIndex(self.labels)
            self.labelIds = sorted(self.label2ind.keys())
            self.num_cats = len(self.labelIds)

            self.labelIds_base = buildLabelIndex(data_base['labels']).keys()
            self.labelIds_novel = buildLabelIndex(data_novel['labels']).keys()
            self.num_cats_base = len(self.labelIds_base)
            self.num_cats_novel = len(self.labelIds_novel)
            intersection = set(self.labelIds_base) & set(self.labelIds_novel)
            assert(len(intersection) == 0)
        else:
            raise ValueError('Not valid phase {0}'.format(self.phase))

        mean_pix = [x/255.0 for x in [120.39586422,  115.59361427, 104.54012653]]
        std_pix = [x/255.0 for x in [70.68188272,  68.27635443,  72.54505529]]
        normalize = transforms.Normalize(mean=mean_pix, std=std_pix)

        if (self.phase=='test' or self.phase=='val') or (do_not_use_random_transf==True):
            self.transform = transforms.Compose([
                lambda x: np.asarray(x),
                transforms.ToTensor(),
                normalize
            ])
        else:
            self.transform = transforms.Compose([
                transforms.RandomCrop(84, padding=8),
                transforms.RandomHorizontalFlip(),
                lambda x: np.asarray(x),
                transforms.ToTensor(),
                normalize
            ])

    def __getitem__(self, index):
        img, label = self.data[index], self.labels[index]
        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)
        return img, label

    def __len__(self):
        return len(self.data)

class FewShotDataloader():
    def __init__(self,
                 dataset,
                 nKnovel=5, # number of novel categories.
                 nKbase=-1, # number of base categories.
                 nExemplars=1, # number of training examples per novel category.
                 nTestNovel=15*5, # number of test examples for all the novel categories.
                 nTestBase=15*5, # number of test examples for all the base categories.
                 batch_size=1, # number of training episodes per batch.
                 num_workers=4,
                 epoch_size=2000, # number of batches per epoch.
                 ):

        self.dataset = dataset
        self.phase = self.dataset.phase
        max_possible_nKnovel = (self.dataset.num_cats_base if self.phase=='train'
                                else self.dataset.num_cats_novel)
        assert(nKnovel >= 0 and nKnovel < max_possible_nKnovel)
        self.nKnovel = nKnovel

        max_possible_nKbase = self.dataset.num_cats_base
        nKbase = nKbase if nKbase >= 0 else max_possible_nKbase
        if self.phase=='train' and nKbase > 0:
            nKbase -= self.nKnovel
            max_possible_nKbase -= self.nKnovel

        assert(nKbase >= 0 and nKbase <= max_possible_nKbase)
        self.nKbase = nKbase

        self.nExemplars = nExemplars
        self.nTestNovel = nTestNovel
        self.nTestBase = nTestBase
        self.batch_size = batch_size
        self.epoch_size = epoch_size
        self.num_workers = num_workers
        self.is_eval_mode = (self.phase=='test') or (self.phase=='val')

    def sampleImageIdsFrom(self, cat_id, sample_size=1):
        """
        Samples `sample_size` number of unique image ids picked from the
        category `cat_id` (i.e., self.dataset.label2ind[cat_id]).

        Args:
            cat_id: a scalar with the id of the category from which images will
                be sampled.
            sample_size: number of images that will be sampled.

        Returns:
            image_ids: a list of length `sample_size` with unique image ids.
        """
        assert(cat_id in self.dataset.label2ind)
        assert(len(self.dataset.label2ind[cat_id]) >= sample_size)
        # Note: random.sample samples elements without replacement.
        return random.sample(self.dataset.label2ind[cat_id], sample_size)

    def sampleCategories(self, cat_set, sample_size=1):
        """
        Samples `sample_size` number of unique categories picked from the
        `cat_set` set of categories. `cat_set` can be either 'base' or 'novel'.

        Args:
            cat_set: string that specifies the set of categories from which
                categories will be sampled.
            sample_size: number of categories that will be sampled.

        Returns:
            cat_ids: a list of length `sample_size` with unique category ids.
        """
        if cat_set=='base':
            labelIds = self.dataset.labelIds_base
        elif cat_set=='novel':
            labelIds = self.dataset.labelIds_novel
        else:
            raise ValueError('Not recognized category set {}'.format(cat_set))

        assert(len(labelIds) >= sample_size)
        # return sample_size unique categories chosen from labelIds set of
        # categories (that can be either self.labelIds_base or self.labelIds_novel)
        # Note: random.sample samples elements without replacement.
        return random.sample(labelIds, sample_size)

    def sample_base_and_novel_categories(self, nKbase, nKnovel):
        """
        Samples `nKbase` number of base categories and `nKnovel` number of novel
        categories.

        Args:
            nKbase: number of base categories
            nKnovel: number of novel categories

        Returns:
            Kbase: a list of length 'nKbase' with the ids of the sampled base
                categories.
            Knovel: a list of lenght 'nKnovel' with the ids of the sampled novel
                categories.
        """
        if self.is_eval_mode:
            assert(nKnovel <= self.dataset.num_cats_novel)
            # sample from the set of base categories 'nKbase' number of base
            # categories.
            Kbase = sorted(self.sampleCategories('base', nKbase))
            # sample from the set of novel categories 'nKnovel' number of novel
            # categories.
            Knovel = sorted(self.sampleCategories('novel', nKnovel))
        else:
            # sample from the set of base categories 'nKnovel' + 'nKbase' number
            # of categories.
            cats_ids = self.sampleCategories('base', nKnovel+nKbase)
            assert(len(cats_ids) == (nKnovel+nKbase))
            # Randomly pick 'nKnovel' number of fake novel categories and keep
            # the rest as base categories.
            random.shuffle(cats_ids)
            Knovel = sorted(cats_ids[:nKnovel])
            Kbase = sorted(cats_ids[nKnovel:])

        return Kbase, Knovel

    def sample_test_examples_for_base_categories(self, Kbase, nTestBase):
        """
        Sample `nTestBase` number of images from the `Kbase` categories.

        Args:
            Kbase: a list of length `nKbase` with the ids of the categories from
                where the images will be sampled.
            nTestBase: the total number of images that will be sampled.

        Returns:
            Tbase: a list of length `nTestBase` with 2-element tuples. The 1st
                element of each tuple is the image id that was sampled and the
                2nd elemend is its category label (which is in the range
                [0, len(Kbase)-1]).
        """
        Tbase = []
        if len(Kbase) > 0:
            # Sample for each base category a number images such that the total
            # number sampled images of all categories to be equal to `nTestBase`.
            KbaseIndices = np.random.choice(
                np.arange(len(Kbase)), size=nTestBase, replace=True)
            KbaseIndices, NumImagesPerCategory = np.unique(
                KbaseIndices, return_counts=True)

            for Kbase_idx, NumImages in zip(KbaseIndices, NumImagesPerCategory):
                imd_ids = self.sampleImageIdsFrom(
                    Kbase[Kbase_idx], sample_size=NumImages)
                Tbase += [(img_id, Kbase_idx) for img_id in imd_ids]

        assert(len(Tbase) == nTestBase)

        return Tbase

    def sample_train_and_test_examples_for_novel_categories(
            self, Knovel, nTestNovel, nExemplars, nKbase):
        """Samples train and test examples of the novel categories.

        Args:
    	    Knovel: a list with the ids of the novel categories.
            nTestNovel: the total number of test images that will be sampled
                from all the novel categories.
            nExemplars: the number of training examples per novel category that
                will be sampled.
            nKbase: the number of base categories. It is used as offset of the
                category index of each sampled image.

        Returns:
            Tnovel: a list of length `nTestNovel` with 2-element tuples. The
                1st element of each tuple is the image id that was sampled and
                the 2nd element is its category label (which is in the range
                [nKbase, nKbase + len(Knovel) - 1]).
            Exemplars: a list of length len(Knovel) * nExemplars of 2-element
                tuples. The 1st element of each tuple is the image id that was
                sampled and the 2nd element is its category label (which is in
                the ragne [nKbase, nKbase + len(Knovel) - 1]).
        """

        if len(Knovel) == 0:
            return [], []

        nKnovel = len(Knovel)
        Tnovel = []
        Exemplars = []
        assert((nTestNovel % nKnovel) == 0)
        nEvalExamplesPerClass = nTestNovel / nKnovel

        for Knovel_idx in range(len(Knovel)):
            imd_ids = self.sampleImageIdsFrom(
                Knovel[Knovel_idx],
                sample_size=(nEvalExamplesPerClass + nExemplars))

            imds_tnovel = imd_ids[:nEvalExamplesPerClass]
            imds_ememplars = imd_ids[nEvalExamplesPerClass:]

            Tnovel += [(img_id, nKbase+Knovel_idx) for img_id in imds_tnovel]
            Exemplars += [(img_id, nKbase+Knovel_idx) for img_id in imds_ememplars]
        assert(len(Tnovel) == nTestNovel)
        assert(len(Exemplars) == len(Knovel) * nExemplars)
        random.shuffle(Exemplars)

        return Tnovel, Exemplars

    def sample_episode(self):
        """Samples a training episode."""
        nKnovel = self.nKnovel
        nKbase = self.nKbase
        nTestNovel = self.nTestNovel
        nTestBase = self.nTestBase
        nExemplars = self.nExemplars

        Kbase, Knovel = self.sample_base_and_novel_categories(nKbase, nKnovel)
        Tbase = self.sample_test_examples_for_base_categories(Kbase, nTestBase)
        Tnovel, Exemplars = self.sample_train_and_test_examples_for_novel_categories(
            Knovel, nTestNovel, nExemplars, nKbase)

        # concatenate the base and novel category examples.
        Test = Tbase + Tnovel
        random.shuffle(Test)
        Kall = Kbase + Knovel

        return Exemplars, Test, Kall, nKbase

    def createExamplesTensorData(self, examples):
        """
        Creates the examples image and label tensor data.

        Args:
            examples: a list of 2-element tuples, each representing a
                train or test example. The 1st element of each tuple
                is the image id of the example and 2nd element is the
                category label of the example, which is in the range
                [0, nK - 1], where nK is the total number of categories
                (both novel and base).

        Returns:
            images: a tensor of shape [nExamples, Height, Width, 3] with the
                example images, where nExamples is the number of examples
                (i.e., nExamples = len(examples)).
            labels: a tensor of shape [nExamples] with the category label
                of each example.
        """
        images = torch.stack(
            [self.dataset[img_idx][0] for img_idx, _ in examples], dim=0)
        labels = torch.LongTensor([label for _, label in examples])
        return images, labels

    def get_iterator(self, epoch=0):
        rand_seed = epoch
        random.seed(rand_seed)
        np.random.seed(rand_seed)
        def load_function(iter_idx):
            Exemplars, Test, Kall, nKbase = self.sample_episode()
            Xt, Yt = self.createExamplesTensorData(Test)
            Kall = torch.LongTensor(Kall)
            if len(Exemplars) > 0:
                Xe, Ye = self.createExamplesTensorData(Exemplars)
                return Xe, Ye, Xt, Yt, Kall, nKbase
            else:
                return Xt, Yt, Kall, nKbase

        tnt_dataset = tnt.dataset.ListDataset(
            elem_list=range(self.epoch_size), load=load_function)
        data_loader = tnt_dataset.parallel(
            batch_size=self.batch_size,
            num_workers=(0 if self.is_eval_mode else self.num_workers),
            shuffle=(False if self.is_eval_mode else True))

        return data_loader

    def __call__(self, epoch=0):
        return self.get_iterator(epoch)

    def __len__(self):
        return (self.epoch_size / self.batch_size)

class ImageNetLowShot(data.Dataset):
    def __init__(self,
                 phase='train',
                 split='train',
                 do_not_use_random_transf=False):
        self.phase = phase
        self.split = split
        assert(phase=='train' or phase=='test' or phase=='val')
        assert(split=='train' or split=='val')
        self.name = 'ImageNetLowShot_Phase_' + phase + '_Split_' + split

        print('Loading ImageNet dataset (for few-shot benchmark) - phase {0}'.
            format(phase))

        #***********************************************************************
        with open(_IMAGENET_LOWSHOT_BENCHMARK_CATEGORY_SPLITS_PATH, 'r') as f:
            label_idx = json.load(f)
        base_classes = label_idx['base_classes']
        novel_classes_val_phase = label_idx['novel_classes_1']
        novel_classes_test_phase = label_idx['novel_classes_2']
        #***********************************************************************

        transforms_list = []
        if (phase!='train') or (do_not_use_random_transf==True):
            transforms_list.append(transforms.Scale(256))
            transforms_list.append(transforms.CenterCrop(224))
        else:
            transforms_list.append(transforms.RandomSizedCrop(224))
            jitter_params = {'Brightness': 0.4, 'Contrast': 0.4, 'Color': 0.4}
            transforms_list.append(ImageJitter(jitter_params))
            transforms_list.append(transforms.RandomHorizontalFlip())

        transforms_list.append(lambda x: np.asarray(x))
        transforms_list.append(transforms.ToTensor())
        mean_pix = [0.485, 0.456, 0.406]
        std_pix = [0.229, 0.224, 0.225]
        transforms_list.append(transforms.Normalize(mean=mean_pix, std=std_pix))

        self.transform = transforms.Compose(transforms_list)

        traindir = os.path.join(_IMAGENET_DATASET_DIR, 'train')
        valdir = os.path.join(_IMAGENET_DATASET_DIR, 'val')
        self.data = datasets.ImageFolder(
            traindir if split=='train' else valdir, self.transform)
        self.labels = [item[1] for item in self.data.imgs]

        self.label2ind = buildLabelIndex(self.labels)
        self.labelIds = sorted(self.label2ind.keys())
        self.num_cats = len(self.labelIds)
        assert(self.num_cats==1000)

        self.labelIds_base = base_classes
        self.num_cats_base = len(self.labelIds_base)
        if self.phase=='val' or self.phase=='test':
            self.labelIds_novel = (
                novel_classes_val_phase if (self.phase=='val') else
                novel_classes_test_phase)
            self.num_cats_novel = len(self.labelIds_novel)

            intersection = set(self.labelIds_base) & set(self.labelIds_novel)
            assert(len(intersection) == 0)

    def __getitem__(self, index):
        img, label = self.data[index]
        return img, label

    def __len__(self):
        return len(self.data)

class ImageNet(data.Dataset):
    def __init__(self, split='train'):
        self.split = split
        assert(split=='train' or split=='val')
        self.name = 'ImageNet_Split_' + split

        print('Loading ImageNet dataset - split {0}'.format(split))
        transforms_list = []
        transforms_list.append(transforms.Scale(256))
        transforms_list.append(transforms.CenterCrop(224))
        transforms_list.append(lambda x: np.asarray(x))
        transforms_list.append(transforms.ToTensor())
        mean_pix = [0.485, 0.456, 0.406]
        std_pix = [0.229, 0.224, 0.225]
        transforms_list.append(transforms.Normalize(mean=mean_pix, std=std_pix))
        self.transform = transforms.Compose(transforms_list)

        traindir = os.path.join(_IMAGENET_DATASET_DIR, 'train')
        valdir = os.path.join(_IMAGENET_DATASET_DIR, 'val')
        self.data = datasets.ImageFolder(
            traindir if split=='train' else valdir, self.transform)
        self.labels = [item[1] for item in self.data.imgs]

    def __getitem__(self, index):
        img, label = self.data[index]
        return img, label

    def __len__(self):
        return len(self.data)

class SimpleDataloader():
    def __init__(self, dataset, batch_size, num_workers=4):
        self.dataset = dataset
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.epoch_size = len(dataset)

    def get_iterator(self):
        def load_fun_(idx):
            img, label = self.dataset[idx]
            return img, label

        tnt_dataset = tnt.dataset.ListDataset(
            elem_list=range(self.epoch_size), load=load_fun_)

        data_loader = tnt_dataset.parallel(
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            shuffle=False, drop_last=False)

        return data_loader

    def __call__(self):
        return self.get_iterator()

    def __len__(self):
        return (self.epoch_size / self.batch_size)

class ImageNetLowShotFeaturesLegacy():
    def __init__(
        self,
        data_dir,
        phase='train',
        add_novel_split='val'):

        self.phase = phase
        assert(phase=='train' or phase=='test' or phase=='val')
        self.name = 'ImageNetLowShotFeatures_Phase_' + phase

        split = 'train' if (phase=='train') else 'val'
        dataset_file = os.path.join(data_dir, 'feature_dataset_'+split+'.json')
        self.data_file = h5py.File(dataset_file, 'r')
        self.count = self.data_file['count'][0]
        self.features = self.data_file['all_features'][...]
        self.labels = self.data_file['all_labels'][:self.count].tolist()

        #***********************************************************************
        with open(_IMAGENET_LOWSHOT_BENCHMARK_CATEGORY_SPLITS_PATH, 'r') as f:
            label_idx = json.load(f)
        base_classes = label_idx['base_classes']
        base_classes_val_phase = label_idx['base_classes_1']
        base_classes_test_phase = label_idx['base_classes_2']
        novel_classes_val_phase = label_idx['novel_classes_1']
        novel_classes_test_phase = label_idx['novel_classes_2']
        #***********************************************************************

        self.label2ind = buildLabelIndex(self.labels)
        self.labelIds = sorted(self.label2ind.keys())
        self.num_cats = len(self.labelIds)
        assert(self.num_cats==1000)

        self.labelIds_base = base_classes
        self.num_cats_base = len(self.labelIds_base)

        novel_split = add_novel_split if (self.phase=='train') else self.phase
        if novel_split=='val' or novel_split=='test':
            self.labelIds_novel = (
                novel_classes_val_phase if (novel_split=='val') else
                novel_classes_test_phase)
            self.num_cats_novel = len(self.labelIds_novel)

            intersection = set(self.labelIds_base) & set(self.labelIds_novel)
            assert(len(intersection) == 0)

            self.base_classes_subset = (
                base_classes_val_phase if (novel_split=='val') else
                base_classes_test_phase)
        else:
            self.base_classes_subset = base_classes

    def __getitem__(self, index):
        features_this = torch.Tensor(self.features[index]).view(-1,1,1)
        label_this = self.labels[index]
        return features_this, label_this

    def __len__(self):
        return int(self.count)

class ImageNetLowShotFeatures():
    def __init__(
        self,
        data_dir, # path to the directory with the saved ImageNet features.
        image_split='train', # the image split of the ImageNet that will be loaded.
        phase='train', # whether the dataset will be used for training, validating, or testing a model.
        ):
        assert(image_split=='train' or image_split=='val')
        assert(phase=='train' or phase=='val' or phase=='test')

        self.phase = phase
        self.image_split = image_split
        self.name = ('ImageNetLowShotFeatures_ImageSplit_' + self.image_split
                     +'_Phase_' + self.phase)

        dataset_file = os.path.join(
            data_dir, 'feature_dataset_' + self.image_split + '.json')
        self.data_file = h5py.File(dataset_file, 'r')
        self.count = self.data_file['count'][0]
        self.features = self.data_file['all_features'][...]
        self.labels = self.data_file['all_labels'][:self.count].tolist()

        #***********************************************************************
        with open(_IMAGENET_LOWSHOT_BENCHMARK_CATEGORY_SPLITS_PATH, 'r') as f:
            label_idx = json.load(f)
        base_classes = label_idx['base_classes']
        base_classes_val_split = label_idx['base_classes_1']
        base_classes_test_split = label_idx['base_classes_2']
        novel_classes_val_split = label_idx['novel_classes_1']
        novel_classes_test_split = label_idx['novel_classes_2']
        #***********************************************************************

        self.label2ind = buildLabelIndex(self.labels)
        self.labelIds = sorted(self.label2ind.keys())
        self.num_cats = len(self.labelIds)
        assert(self.num_cats==1000)

        self.labelIds_base = base_classes
        self.num_cats_base = len(self.labelIds_base)

        if self.phase=='val' or self.phase=='test':
            self.labelIds_novel = (
                novel_classes_val_split if (self.phase=='val') else
                novel_classes_test_split)
            self.num_cats_novel = len(self.labelIds_novel)

            intersection = set(self.labelIds_base) & set(self.labelIds_novel)
            assert(len(intersection) == 0)
            self.base_classes_eval_split = (
                base_classes_val_split if (self.phase=='val') else
                base_classes_test_split)
            self.base_classes_subset = self.base_classes_eval_split

    def __getitem__(self, index):
        features_this = torch.Tensor(self.features[index]).view(-1,1,1)
        label_this = self.labels[index]
        return features_this, label_this

    def __len__(self):
        return int(self.count)

class LowShotDataloader():
    def __init__(
        self,
        dataset_train_novel,
        dataset_evaluation,
        nExemplars=1,
        batch_size=1,
        num_workers=4):

        self.nExemplars = nExemplars
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.dataset_train_novel = dataset_train_novel
        self.dataset_evaluation = dataset_evaluation

        assert(self.dataset_evaluation.labelIds_novel ==
               self.dataset_train_novel.labelIds_novel)

        assert(self.dataset_evaluation.labelIds_base ==
               self.dataset_train_novel.labelIds_base)

        assert(self.dataset_evaluation.base_classes_eval_split ==
               self.dataset_train_novel.base_classes_eval_split)

        self.nKnovel = self.dataset_evaluation.num_cats_novel
        self.nKbase = self.dataset_evaluation.num_cats_base

        # Category ids of the base categories.
        self.Kbase = sorted(self.dataset_evaluation.labelIds_base)
        assert(self.nKbase == len(self.Kbase))
        # Category ids of the novel categories.
        self.Knovel = sorted(self.dataset_evaluation.labelIds_novel)
        assert(self.nKnovel == len(self.Knovel))

        self.Kall = self.Kbase + self.Knovel

        self.CategoryId2LabelIndex = {
            category_id: label_index for label_index, category_id in enumerate(self.Kall)
        }
        self.Kbase_eval_split = self.dataset_train_novel.base_classes_eval_split

        Kbase_set = set(self.Kall[:self.nKbase])
        Kbase_eval_split_set = set(self.Kbase_eval_split)
        assert(len(set.intersection(Kbase_set, Kbase_eval_split_set)) == len(Kbase_eval_split_set))

        self.base_eval_split_labels = sorted(
            [self.CategoryId2LabelIndex[category_id] for category_id in self.Kbase_eval_split]
        )

        # Collect the image indices of the evaluation set for both the base and
        # the novel categories.
        data_indices = []
        for category_id in self.Kbase_eval_split:
            data_indices += self.dataset_evaluation.label2ind[category_id]
        for category_id in self.Knovel:
            data_indices += self.dataset_evaluation.label2ind[category_id]
        self.eval_data_indices = sorted(data_indices)
        self.epoch_size = len(self.eval_data_indices)

    def base_category_label_indices(self):
        return self.base_eval_split_labels

    def novel_category_label_indices(self):
        return range(self.nKbase, len(self.Kall))

    def sampleImageIdsFrom(self, cat_id, sample_size=1):
        """
        Samples `sample_size` number of unique image ids picked from the
        category `cat_id` (i.e., self.dataset_train_novel.label2ind[cat_id]).

        Args:
            cat_id: a scalar with the id of the category from which images will
                be sampled.
            sample_size: number of images that will be sampled.

        Returns:
            image_ids: a list of length `sample_size` with unique image ids.
        """
        assert(cat_id in self.dataset_train_novel.label2ind)
        assert(len(self.dataset_train_novel.label2ind[cat_id]) >= sample_size)
        # Note: random.sample samples elements without replacement.
        return random.sample(self.dataset_train_novel.label2ind[cat_id], sample_size)

    def sample_training_examples_for_novel_categories(
        self, Knovel, nExemplars, nKbase):
        """Samples (a few) training examples for the novel categories.

        Args:
            Knovel: a list with the ids of the novel categories.
            nExemplars: the number of training examples per novel category.
            nKbase: the number of base categories.

        Returns:
            Exemplars: a list of length len(Knovel) * nExemplars of 2-element
                tuples. The 1st element of each tuple is the image id that was
                sampled and the 2nd element is its category label (which is in
                the ragne [nKbase, nKbase + len(Knovel) - 1]).
        """
        Exemplars = []
        for knovel_idx, knovel_label in enumerate(Knovel):
            imds = self.sampleImageIdsFrom(knovel_label, sample_size=nExemplars)
            Exemplars += [(img_id, nKbase + knovel_idx) for img_id in imds]
        random.shuffle(Exemplars)

        return  Exemplars

    def create_examples_tensor_data(self, examples):
        """
        Creates the examples image and label tensor data.

        Args:
            examples: a list of 2-element tuples, each representing a
                train or test example. The 1st element of each tuple
                is the image id of the example and 2nd element is the
                category label of the example, which is in the range
                [0, nK - 1], where nK is the total number of categories
                (both novel and base).

        Returns:
            images: a tensor of shape [nExamples, Height, Width, 3] with the
                example images, where nExamples is the number of examples
                (i.e., nExamples = len(examples)).
            labels: a tensor of shape [nExamples] with the category label
                of each example.
        """
        images = torch.stack(
            [self.dataset_train_novel[img_idx][0] for img_idx, _ in examples],
            dim=0)
        labels = torch.LongTensor([label for _, label in examples])
        return images, labels

    def sample_training_data_for_novel_categories(self, exp_id=0):
        nKnovel = self.nKnovel
        nKbase = self.nKbase
        random.seed(exp_id) # fix the seed for this experiment.
        # Sample `nExemplars` number of training examples per novel category.
        train_examples = self.sample_training_examples_for_novel_categories(
            self.Knovel, self.nExemplars, nKbase)
        Kall = torch.LongTensor(self.Kall)
        images_train, labels_train = self.create_examples_tensor_data(
            train_examples)

        return images_train, labels_train, Kall, nKbase, nKnovel

    def get_iterator(self, epoch=0):
        def load_fun_(idx):
            img_idx = self.eval_data_indices[idx]
            img, category_id = self.dataset_evaluation[img_idx]
            label = (self.CategoryId2LabelIndex[category_id]
                     if (category_id in self.CategoryId2LabelIndex) else -1)
            return img, label

        tnt_dataset = tnt.dataset.ListDataset(
            elem_list=range(self.epoch_size), load=load_fun_)
        data_loader = tnt_dataset.parallel(
            batch_size=self.batch_size, num_workers=self.num_workers,
            shuffle=False, drop_last=False)
        return data_loader

    def __call__(self, epoch=0):
        return self.get_iterator(epoch)

    def __len__(self):
        return int(math.ceil(float(self.epoch_size)/self.batch_size))

class LowShotDataloaderLegacy():
    def __init__(
        self,
        dataset_train_novel,
        dataset_evaluation,
        nExemplars=1,
        batch_size=1,
        num_workers=4):

        self.dataset_train_novel  = dataset_train_novel
        self.dataset_evaluation = dataset_evaluation

        assert(self.dataset_evaluation.labelIds_novel ==
               self.dataset_train_novel.labelIds_novel)

        # Collect the image indices of the evaluation set for both the base and
        # the novel categories.
        data_inds = []
        for kid in self.dataset_evaluation.labelIds_base:
            data_inds += self.dataset_evaluation.label2ind[kid]
        for kid in self.dataset_evaluation.labelIds_novel:
            data_inds += self.dataset_evaluation.label2ind[kid]
        self.eval_data_indices = sorted(data_inds)

        self.nKnovel = self.dataset_evaluation.num_cats_novel
        self.nKbase = self.dataset_evaluation.num_cats_base

        self.nExemplars = nExemplars
        self.batch_size = batch_size
        self.epoch_size = len(self.eval_data_indices)
        self.num_workers = num_workers

    def sampleImageIdsFrom(self, cat_id, sample_size=1):
        """
        Samples `sample_size` number of unique image ids picked from the
        category `cat_id` (i.e., self.dataset_train_novel.label2ind[cat_id]).

        Args:
            cat_id: a scalar with the id of the category from which images will
                be sampled.
            sample_size: number of images that will be sampled.

        Returns:
            image_ids: a list of length `sample_size` with unique image ids.
        """
        assert(cat_id in self.dataset_train_novel.label2ind)
        assert(len(self.dataset_train_novel.label2ind[cat_id]) >= sample_size)
        # Note: random.sample samples elements without replacement.
        return random.sample(self.dataset_train_novel.label2ind[cat_id], sample_size)

    def sampleCategories(self, cat_set, sample_size=1):
        """
        Samples `sample_size` number of unique categories picked from the
        `cat_set` set of categories. `cat_set` can be either 'base' or 'novel'.

        Args:
            cat_set: string that specifies the set of categories from which
                categories will be sampled.
            sample_size: number of categories that will be sampled.

        Returns:
            cat_ids: a list of length `sample_size` with unique category ids.
        """
        if cat_set=='base':
            labelIds = self.dataset_evaluation.labelIds_base
        elif cat_set=='novel':
            labelIds = self.dataset_evaluation.labelIds_novel
        else:
            raise ValueError('Not recognized category set {}'.format(cat_set))

        assert(len(labelIds) >= sample_size)
        # return sample_size unique categories chosen from labelIds set of
        # categories (that can be either self.labelIds_base or self.labelIds_novel)
        # Note: random.sample samples elements without replacement.
        return random.sample(labelIds, sample_size)

    def sample_novel_data(self):
        """Samples a few training examples for each novel category."""
        nKnovel = self.nKnovel
        nKbase = self.nKbase
        nExemplars = self.nExemplars

        # Kbase = sorted(self.dataset_evaluation.labelIds_base)
        # Knovel = sorted(self.dataset_evaluation.labelIds_novel)
        Kbase = sorted(self.sampleCategories('base', nKbase))
        Knovel = sorted(self.sampleCategories('novel', nKnovel))

        Exemplars = []
        for knovel_idx, knovel_label in enumerate(Knovel):
            imds = self.sampleImageIdsFrom(knovel_label, sample_size=nExemplars)
            Exemplars += [(img_id, nKbase + knovel_idx) for img_id in imds]
        random.shuffle(Exemplars)

        Kids = Kbase + Knovel

        return  Exemplars, Kids, nKbase, nKnovel

    def create_examples_tensor_data(self, examples):
        """
        Creates the examples image and label tensor data.

        Args:
            examples: a list of 2-element tuples, each representing a
                train or test example. The 1st element of each tuple
                is the image id of the example and 2nd element is the
                category label of the example, which is in the range
                [0, nK - 1], where nK is the total number of categories
                (both novel and base).

        Returns:
            images: a tensor of shape [nExamples, Height, Width, 3] with the
                example images, where nExamples is the number of examples
                (i.e., nExamples = len(examples)).
            labels: a tensor of shape [nExamples] with the category label
                of each example.
        """
        images = torch.stack(
            [self.dataset_train_novel[img_idx][0] for img_idx, _ in examples],
            dim=0)
        labels = torch.LongTensor([label for _, label in examples])
        return images, labels

    def getNovelCategoriesTrainingData(self, exp_id=0):
        random.seed(exp_id) # fix the seed for this experiment.

        # Sample training examples for each novel category.
        Exemplars, Kids, nKbase, nKnovel = self.sample_novel_data()
        self.Kid2Label = {kid: label_idx for label_idx, kid in enumerate(Kids)}

        base_classes_subset = self.dataset_train_novel.base_classes_subset
        assert(len(set.intersection(set(Kids[:nKbase]),set(base_classes_subset))) == len(base_classes_subset))
        self.Kids_base_subset = sorted([self.Kid2Label[kid] for kid in base_classes_subset])

        Kids = torch.LongTensor(Kids)
        Xe, Ye = self.create_examples_tensor_data(Exemplars)
        return Xe, Ye, Kids, nKbase, nKnovel

    def sample_training_data_of_novel_categories(self, exp_id=0):
        nKnovel = self.nKnovel
        nKbase = self.nKbase
        nExemplars = self.nExemplars

        random.seed(exp_id) # fix the seed for this experiment.
        breakpoint()
        # Ids of the base categories.
        Kbase = sorted(self.dataset_evaluation.labelIds_base)
        # Ids of the novel categories.
        Knovel = sorted(self.dataset_evaluation.labelIds_novel)
        assert(len(Kbase) == nKnovel and len(Knovel) == nKbase)
        Kall = Kbase + Knovel

        # Sample `nExemplars` number of training examples for each novel
        # category.
        train_examples = self.sample_training_examples_for_novel_categories(
            Knovel, nExemplars)

        breakpoint()
        self.Kid2Label = {kid: label_idx for label_idx, kid in enumerate(Kall)}

        breakpoint()
        base_classes_subset = self.dataset_train_novel.base_classes_subset
        assert(len(set.intersection(set(Kall[:nKbase]),set(base_classes_subset))) == len(base_classes_subset))
        self.Kids_base_subset = sorted([self.Kid2Label[kid] for kid in base_classes_subset])

        Kall = torch.LongTensor(Kall)
        images_train, labels_train = self.create_examples_tensor_data(train_examples)
        return images_train, labels_train, Kall, nKbase, nKnovel

    def get_iterator(self, epoch=0):
        def load_fun_(idx):
            img_idx = self.eval_data_indices[idx]
            img, kid = self.dataset_evaluation[img_idx]
            label = self.Kid2Label[kid] if (kid in self.Kid2Label) else -1
            return img, label

        tnt_dataset = tnt.dataset.ListDataset(
            elem_list=range(self.epoch_size), load=load_fun_)
        data_loader = tnt_dataset.parallel(
            batch_size=self.batch_size, num_workers=self.num_workers,
            shuffle=False, drop_last=False)
        return data_loader

    def __call__(self, epoch=0):
        return self.get_iterator(epoch)

    def __len__(self):
        return int(math.ceil(float(self.epoch_size)/self.batch_size))


class Solution:
    def combine(self, n: int, k: int) -> List[List[int]]:

        res= []
        def solve(curr,k,arr):
            if k==0:
                res.append(arr[::])
                return

            if curr>n:
                return

            #taking the curr
            arr.append(curr)
            solve(curr+1,k-1,arr)

            # not taking it
            arr.pop()
            solve(curr+1,k,arr)

        solve(1,k,[])
        return res


def format_phone_number(func):
    def inner(phone):
        result = func(phone)
        if len(result)<11:
            result = '+38'+result
        else:
            result = '+'+result
        return result
    return inner

@format_phone_number
def sanitize_phone_number(phone):
    new_phone = (
        phone.strip()
            .removeprefix("+")
            .replace("(", "")
            .replace(")", "")
            .replace("-", "")
            .replace(" ", "")
    )
    return new_phone


import csv
import nltk
from langdetect import detect

def unique(list1):
    # intilize a null list
    unique_list = []

    # traverse for all elements
    for x in list1:
        # check if exists in unique_list or not
        if x not in unique_list:
            unique_list.append(x)

    return unique_list

def sentenceWordRelation(sentence, wrdList, rvNo, feature,keywordLang):
    stemmerDe = nltk.stem.cistem.Cistem()
    stemmerEn = nltk.SnowballStemmer("english")
    itsGerman = True
    try:
        if detect(sentence) == 'en':
            itsGerman = False
    except:
        itsGerman = True

    itsGermanKeyword = True
    if keywordLang == "en:":
        itsGermanKeyword = False
    else:
        itsGermanKeyword = True

    word_tokens = nltk.tokenize.word_tokenize(sentence)
    tokensDe = []
    tokensEn = []
    for token in word_tokens:
        if '.' in token:
            tokensDe.append(stemmerDe.stem(token.split('.')[0]).lower())
            tokensDe.append(stemmerDe.stem(token.split('.')[1]).lower())
            tokensEn.append(stemmerEn.stem(token.split('.')[0]).lower())
            tokensEn.append(stemmerEn.stem(token.split('.')[1]).lower())
        else:
            tokensDe.append(stemmerDe.stem(token).lower())
            tokensEn.append(stemmerEn.stem(token).lower())

    if itsGerman == True:
        cntDe = 0
        for tokenDe in tokensDe:
            if itsGermanKeyword == True:
                for wrd in wrdList:
                    wrd = wrd.replace("de:","")
                    if stemmerDe.stem(wrd) == tokenDe:
                        cntDe = cntDe + 1
                if cntDe == len(wrdList):
                    # print("review number: " + str(rvNo) + " belonging to feature " + feature.upper() + " and words are: " + str(wrdList))
                    return True
    else:
        cntEn = 0
        for tokenEn in tokensEn:
            if itsGermanKeyword == False:
                for wrd in wrdList:
                    wrd = wrd.replace("en:", "")
                    if stemmerEn.stem(wrd) == tokenEn:
                        cntEn = cntEn + 1
                if cntEn == len(wrdList):
                    print("review number: " + str(rvNo) + " belonging to feature " + feature.upper() + " and words are: " + str(wrdList))
                    return True

csvFileName = 'KeywordsTable_output.csv'
keywordsTable = list(csv.reader(open(csvFileName),delimiter=',')) # CSV file to 2 dimensional list of string

csvFileName = 'Master_Data_Milestone1_temp.csv'
masterData = list(csv.reader(open(csvFileName),delimiter='|')) # CSV file to 2 dimensional list of string

features = [i[0] for i in keywordsTable]

features = unique(features)

outFile = []

masterData[0] = masterData[0] + features

outFile.append(masterData[0])

# for i in range(49,50):
for i in range(1,len(masterData)):
    sentence = masterData[i][9]
    scoreList = []
    for j in range(len(features)):
        score = 0
        for k in range(len(keywordsTable)):
            if features[j] == keywordsTable[k][0]:
                for l in range(1,len(keywordsTable[k])):
                    wrdList = keywordsTable[k][l].split(' ')
                    result = sentenceWordRelation(sentence,wrdList,i,features[j],keywordsTable[k][l][0:3])
                    if result == True:
                        score = score + 1
                        break
                if result == True:
                    break
        scoreList.append(score)
    masterData[i] = masterData[i] + scoreList
    outFile.append(masterData[i])

    if i%10 == 0:
        print(str(i) + " reviews processed.")

masterData_outputFileName = 'Master_Data_Features_Milestone1_temp.csv'
masterData_outputFile = open(masterData_outputFileName, "w", newline='', encoding='utf-8')
csv_out = csv.writer(masterData_outputFile, delimiter='|')
csv_out.writerows(outFile)


from django.urls import path

from .views import (AddProductView, CartDetailView, RemoveProductView,
                    ChangeQuantityView, PriceChangedView)

app_name = 'shoppingcart'

urlpatterns = [
    path('add_product/', AddProductView.as_view(), name='add-product'),
    path('remove_product/', RemoveProductView.as_view(),
         name='remove-product'),
    path('update_quantity/', ChangeQuantityView.as_view(),
         name='update-quantity'),
    path('price_changed/', PriceChangedView.as_view(), name='price-changed'),
    path('detail/', CartDetailView.as_view(), name='cart-detail')
]


from django.shortcuts import render,redirect
from .models import Address
from django.forms import ModelForm
from billing.models import BillingProfile
from .forms import AddressForm
from django.http import Http404

def checkout_address_create_view(request):

    if request.method == 'POST':
        form1 = AddressForm(request.POST,prefix='form1')
        form2 = AddressForm(request.POST,prefix='form2')
        # import pdb; pdb.set_trace()
        if form1.is_valid():
            # print(request.POST)
            instance = form1.save(commit=False)
            billing_profile, billing_profile_created = BillingProfile.objects.new_or_get(request)
            print('hii',billing_profile)
            if billing_profile is not None:
                # address_type = request.POST.get('address_type', 'shipping')
                instance.billing_profile = billing_profile
                instance.address_type = 'shipping'
                instance.save()
                # request.session[address_type + "_address_id"] = instance.id
                #print(address_type + "_address_id")
            else:
                print("Error here")
                return redirect("home")

        if form2.is_valid():
            # print(request.POST)
            instance = form2.save(commit=False)
            billing_profile, billing_profile_created = BillingProfile.objects.new_or_get(request)
            if billing_profile is not None:
                # address_type = request.POST.get('address_type', 'shipping')
                instance.billing_profile = billing_profile
                instance.address_type = 'billing'
                instance.save()
                # request.session[address_type + "_address_id"] = instance.id
               # print(address_type + "_address_id")
                import pdb; pdb.set_trace()
                if instance.address_type != None:
                    return redirect('order:overview')
            else:
                print("Error here")
                return redirect("home")

        # else:
        #
        #     raise Http404

    else:
        form1 = AddressForm(prefix='form1')
        form2 = AddressForm(prefix='form2')
    return render(request,"checkout.html",{"form1":form1,"form2":form2})

def address_update(request):
    billing_profile, billing_profile_created = BillingProfile.objects.new_or_get(request)
    print(billing_profile)
    address_qs = None
    if billing_profile is not None:
        if request.user.is_authenticated():
            address_qs = Address.objects.filter(billing_profile=billing_profile)
        try:
            shipping_address = Address.objects.get(billing_profile=billing_profile.id,address_type='shipping')
        except Address.DoesNotExist:
            print("Show message to user, Address is gone?")
            return redirect("cart:cart")
        try:
            billing_address = Address.objects.get(billing_profile=billing_profile.id,address_type='billing')
        except Address.DoesNotExist:
            print(" Address is gone?")
            return redirect("cart:cart")
        # shipping_address = Address.objects.get(id=27)
        # import pdb; pdb.set_trace()
        form1=AddressForm(prefix='form1',instance=shipping_address,data=request.POST or None )
        form2=AddressForm(prefix='form2',instance=billing_address,data=request.POST or None)

        if request.method=='POST':
            import pdb; pdb.set_trace()
            if form2.is_valid():
                form2.save()
            if form1.is_valid():
                form1.save()
                return redirect('address')

    return render(request,'checkout.html',{'form1':form1,'form2':form2})


import datetime
from pathlib import Path

def check_today_date_in_file(file_path):

    today_date = datetime.datetime.now().date()

    with open(file_path, 'r') as file:
        dates_list = [line.strip() for line in file]

    if str(today_date) in dates_list:
        return True

    else:
        return False

def write_dates_to_file(date, file_path):
    with open(file_path, 'w') as file:
        date_str = date.strftime('%Y-%m-%d')
        file.write(date_str + '\n')

def create_empty_text_file(file_path):
    path = Path(file_path)
    path.touch()


import itertools
import re
import logging

from collections import Counter, OrderedDict
from collections import defaultdict

from typ import python_2_3_compat
from typ.json_results import ResultType

_EXPECTATION_MAP = {
    'Crash': ResultType.Crash,
    'Failure': ResultType.Failure,
    'Pass': ResultType.Pass,
    'Timeout': ResultType.Timeout,
    'Skip': ResultType.Skip
}

RESULT_TAGS = {
    ResultType.Failure: 'Failure',
    ResultType.Crash: 'Crash',
    ResultType.Timeout: 'Timeout',
    ResultType.Pass: 'Pass',
    ResultType.Skip: 'Skip'
}

_SLOW_TAG = 'Slow'
_RETRY_ON_FAILURE_TAG = 'RetryOnFailure'

VALID_RESULT_TAGS = set(
    list(_EXPECTATION_MAP.keys()) +
    [_SLOW_TAG, _RETRY_ON_FAILURE_TAG])

class ConflictResolutionTypes(object):
    UNION = 1
    OVERRIDE = 2

def _group_to_string(group):
    msg = ', '.join(group)
    k = msg.rfind(', ')
    return msg[:k] + ' and ' + msg[k+2:] if k != -1 else msg

def _default_tags_conflict(t1, t2):
    return t1 != t2

class ParseError(Exception):

    def __init__(self, lineno, msg):
        super(ParseError, self).__init__('%d: %s' % (lineno, msg))

class Expectation(object):
    def __init__(self, reason=None, test='*', tags=None, results=None, lineno=0,
                 retry_on_failure=False, is_slow_test=False,
                 conflict_resolution=ConflictResolutionTypes.UNION, raw_tags=None, raw_results=None,
                 is_glob=False, trailing_comments=None):
        """Constructor for expectations.

        Args:
          reason: String that indicates the reason for the expectation.
          test: String indicating which test is being affected.
          tags: List of tags that the expectation applies to. Tags are combined
              using a logical and, i.e., all of the tags need to be present for
              the expectation to apply. For example, if tags = ['Mac', 'Debug'],
              then the test must be running with the 'Mac' and 'Debug' tags
              set; just 'Mac', or 'Mac' and 'Release', would not qualify.
          results: List of outcomes for test. Example: ['Skip', 'Pass']
        """
        tags = tags or []
        self._is_default_pass = not results
        results = results or {ResultType.Pass}
        reason = reason or ''
        trailing_comments = trailing_comments or ''
        assert python_2_3_compat.is_str(reason)
        assert python_2_3_compat.is_str(test)
        self._reason = reason
        self._test = test
        self._tags = frozenset(tags)
        self._results = frozenset(results)
        self._lineno = lineno
        self._raw_tags = raw_tags
        self._raw_results = raw_results
        self.should_retry_on_failure = retry_on_failure
        self.is_slow_test = is_slow_test
        self.conflict_resolution = conflict_resolution
        self._is_glob = is_glob
        self._trailing_comments = trailing_comments

    def __eq__(self, other):
        return (self.reason == other.reason and self.test == other.test
                and self.should_retry_on_failure == other.should_retry_on_failure
                and self.is_slow_test == other.is_slow_test
                and self.tags == other.tags and self.results == other.results
                and self.lineno == other.lineno
                and self.trailing_comments == other.trailing_comments)

    def _set_string_value(self):
        """This method will create an expectation line in string form and set the
        _string_value member variable to it. If the _raw_results lists and _raw_tags
        list are not set then the _tags list and _results set will be used to set them.
        Setting the _raw_results and _raw_tags list to the original lists through the constructor
        during parsing stops unintended modifications to test expectations when rewriting files.
        """
        # If this instance is for a glob type expectation then do not escape
        # the last asterisk
        if self.is_glob:
            assert len(self._test) and self._test[-1] == '*', (
                'For Expectation instances for glob type expectations, the test value '
                'must end in an asterisk')
            pattern = self._test[:-1].replace('*', '\\*') + '*'
        else:
            pattern = self._test.replace('*', '\\*')
        pattern = uri_encode_spaces(pattern)
        self._string_value = ''
        if self._reason:
            self._string_value += self._reason + ' '
        if self.raw_tags:
            self._string_value += '[ %s ] ' % ' '.join(sorted(self.raw_tags))
        self._string_value += pattern + ' '
        self._string_value += '[ %s ]' % ' '.join(sorted(self.raw_results))
        if self._trailing_comments:
            self._string_value += self._trailing_comments

    def add_expectations(self, results, reason=None):
        if reason:
            self._reason = ' '.join(sorted(set(self._reason.split() + reason.split())))
        if not results <= self._results:
            self._results = frozenset(self._results | results)
            self._raw_results = sorted(
                [RESULT_TAGS[t] for t in self._results])

    @property
    def raw_tags(self):
        if not self._raw_tags:
            self._raw_tags = {t[0].upper() + t[1:].lower() for t in self._tags}
        return self._raw_tags

    @property
    def raw_results(self):
        if not self._raw_results:
            self._raw_results = {RESULT_TAGS[t] for t in self._results}
            if self.is_slow_test:
                self._raw_results.add(_SLOW_TAG)
            if self.should_retry_on_failure:
                self._raw_results.add(_RETRY_ON_FAILURE_TAG)
        return self._raw_results

    def to_string(self):
        self._set_string_value()
        return self._string_value

    @property
    def is_default_pass(self):
        return self._is_default_pass

    @property
    def reason(self):
        return self._reason

    @property
    def test(self):
        return self._test

    @test.setter
    def test(self, v):
        if not len(v):
            raise ValueError('Cannot set test to empty string')
        if self.is_glob and v[-1] != '*':
            raise ValueError(
                'test value for glob type expectations must end with an asterisk')
        self._test = v

    @property
    def tags(self):
        return self._tags

    @property
    def results(self):
        return self._results

    @property
    def lineno(self):
        return self._lineno

    @lineno.setter
    def lineno(self, lineno):
        self._lineno = lineno

    @property
    def is_glob(self):
        return self._is_glob

    @property
    def trailing_comments(self):
        return self._trailing_comments

class TaggedTestListParser(object):
    """Parses lists of tests and expectations for them.

    This parser covers the 'tagged' test lists format in:
        bit.ly/chromium-test-list-format

    Takes raw expectations data as a string read from the expectation file
    in the format:

      # This is an example expectation file.
      #
      # tags: [
      #   Mac Mac10.1 Mac10.2
      #   Win Win8
      # ]
      # tags: [ Release Debug ]

      crbug.com/123 [ Win ] benchmark/story [ Skip ]
      ...
    """
    CONFLICTS_ALLOWED = '# conflicts_allowed: '
    CONFLICT_RESOLUTION = '# conflict_resolution: '
    RESULT_TOKEN = '# results: ['
    TAG_TOKEN = '# tags: ['
    # The bug field (optional), including optional subproject.
    BUG_PREFIX_REGEX = '(?:crbug.com/|skbug.com/|webkit.org/)'
    _MATCH_STRING = r'^(?:(' + BUG_PREFIX_REGEX + r'(?:[^/]*/)?\d+\s)*)'
    _MATCH_STRING += r'(?:\[ (.+) \] )?'  # The label field (optional).
    _MATCH_STRING += r'(\S+) '  # The test path field.
    _MATCH_STRING += r'\[ ([^\[.]+) \]'  # The expectation field.
    _MATCH_STRING += r'(\s+#.*)?$'  # End comment (optional).
    MATCHER = re.compile(_MATCH_STRING)

    def __init__(self, raw_data, conflict_resolution=ConflictResolutionTypes.UNION):
        self.tag_sets = set()
        self.conflicts_allowed = False
        self.expectations = []
        self._allowed_results = set()
        self._tag_to_tag_set = {}
        self.conflict_resolution = conflict_resolution
        self._parse_raw_expectation_data(raw_data)

    def _parse_raw_expectation_data(self, raw_data):
        all_lines = raw_data.splitlines()
        lineno = 1
        tag_sets_intersection = set()
        first_tag_line = None
        while lineno <= len(all_lines):
            line = all_lines[lineno - 1].strip()
            if (line.startswith((self.TAG_TOKEN, self.RESULT_TOKEN))):
                if not first_tag_line:
                    first_tag_line = lineno
                tag_sets_intersection.update(
                    self._parse_header_token_line(lineno, line, all_lines))
            elif line.startswith(self.CONFLICT_RESOLUTION):
                self._parse_conflict_resolution_line(lineno, line)
            elif line.startswith(self.CONFLICTS_ALLOWED):
                self._parse_conflicts_allowed_line(lineno, line)
            elif line.startswith('#') or not line:
                # Ignore, it is just a comment or empty.
                lineno += 1
                continue
            elif not tag_sets_intersection:
                self.expectations.append(
                    self._parse_expectation_line(lineno, line))
            else:
                break
            lineno += 1
        if tag_sets_intersection:
            is_multiple_tags = len(tag_sets_intersection) > 1
            tag_tags = 'tags' if is_multiple_tags else 'tag'
            was_were = 'were' if is_multiple_tags else 'was'
            error_msg = 'The {0} {1} {2} found in multiple tag sets'.format(
                tag_tags, _group_to_string(
                    sorted(list(tag_sets_intersection))), was_were)
            raise ParseError(first_tag_line, error_msg)

    def _parse_header_token_line(self, lineno, line, all_lines):
        """Helper function for parsing lines that start with header tokens.

        Returns:
            A set of strings containing any tag set intersections found while
            parsing the given line.
        """
        if self.expectations:
            raise ParseError(lineno,
                             'Tag found after first expectation.')
        if line.startswith(self.TAG_TOKEN):
            token = self.TAG_TOKEN
        else:
            token = self.RESULT_TOKEN

        tag_counts = self._get_tag_counts_from_header_line(
            lineno, line, all_lines, token)
        tag_set = set(tag_counts.keys())
        duplicate_tags = {tag for tag, count in tag_counts.items() if count > 1}

        if duplicate_tags:
            message = 'duplicate tag(s): {}'.format(
                ', '.join(sorted(duplicate_tags)))
            raise ParseError(lineno, message)

        tag_sets_intersection = set()
        if token == self.TAG_TOKEN:
            tag_set = frozenset([t.lower() for t in tag_set])
            tag_sets_intersection.update(
                (t for t in tag_set if t in self._tag_to_tag_set))
            self.tag_sets.add(tag_set)
            self._tag_to_tag_set.update(
                {tg: id(tag_set) for tg in tag_set})
        else:
            for t in tag_set:
                if t not in VALID_RESULT_TAGS:
                    raise ParseError(
                        lineno,
                        'Result tag set [%s] contains values not in '
                        'the list of known values [%s]' %
                        (', '.join(tag_set),
                         ', '.join(VALID_RESULT_TAGS)))
            self._allowed_results.update(tag_set)
        return tag_sets_intersection

    def _get_tag_counts_from_header_line(self, lineno, line, all_lines, token):
        """Helper function for parsing tags from a header line.

        Returns:
            A Counter object containing counts for each found tag.
        """
        right_bracket = line.find(']')
        prefix_size = len(token)
        tag_counts = Counter()
        # Loop through every line until we find the closing ], adding any tags
        # we fine. The line with the closing ] is handled after the loop.
        while lineno <= len(all_lines) and right_bracket == -1:
            tag_counts.update(line[prefix_size:].split())
            lineno += 1
            line = all_lines[lineno - 1].strip()
            prefix_size = 1
            if line[0] != '#':
                raise ParseError(
                    lineno,
                    'Multi-line tag set missing leading "#"')
            right_bracket = line.find(']')

        if line[right_bracket+1:]:
            raise ParseError(
                lineno,
                'Nothing is allowed after a closing tag '
                'bracket')

        tag_counts.update(line[prefix_size:right_bracket].split())
        return tag_counts

    def _parse_conflict_resolution_line(self, lineno, line):
        """Helper function for parsing conflict resolution annotations."""
        value = line[len(self.CONFLICT_RESOLUTION):].lower()
        if value not in ('union', 'override'):
            raise ParseError(
                lineno,
                ("Unrecognized value '%s' given for conflict_resolution"
                 "descriptor" %
                 value))
        if value == 'union':
            self.conflict_resolution = ConflictResolutionTypes.UNION
        else:
            self.conflict_resolution = ConflictResolutionTypes.OVERRIDE

    def _parse_conflicts_allowed_line(self, lineno, line):
        """Helper function for parsing conflicts allowed annotations."""
        bool_value = line[len(self.CONFLICTS_ALLOWED):].lower()
        if bool_value not in ('true', 'false'):
            raise ParseError(
                lineno,
                ("Unrecognized value '%s' given for conflicts_allowed "
                 "descriptor" %
                 bool_value))
        self.conflicts_allowed = bool_value == 'true'

    def _parse_expectation_line(self, lineno, line):
        reason, raw_tags, test, raw_results, trailing_comments =\
            self._parse_expectation_line_into_components(lineno, line)
        tags = [raw_tag.lower()
                for raw_tag in raw_tags.split()] if raw_tags else []
        self._validate_expectation_structure(lineno, test, tags)
        results, retry_on_failure, is_slow_test =\
            self._parse_and_validate_raw_results(lineno, raw_results)

        # replace %20 in test path to ' '
        test = uri_decode_spaces(test)

        # remove escapes for asterisks
        is_glob = not test.endswith('\\*') and test.endswith('*')
        test = test.replace('\\*', '*')
        if raw_tags:
            raw_tags = raw_tags.split()
        if raw_results:
            raw_results = raw_results.split()
        # Tags from tag groups will be stored in lower case in the Expectation
        # instance. These tags will be compared to the tags passed in to
        # the Runner instance which are also stored in lower case.
        return Expectation(
            reason, test, tags, results, lineno, retry_on_failure, is_slow_test,
            self.conflict_resolution, raw_tags=raw_tags, raw_results=raw_results,
            is_glob=is_glob, trailing_comments=trailing_comments)

    def _parse_expectation_line_into_components(self, lineno, line):
        """Helper function to break a single expectation line into components.

        Returns:
            A tuple of strings (reason, raw_tags, test, raw_results, trailing
            comments) which correspond to the components that make up an
            expectation line in an expectation file. Values may be empty strings
            if that particular component was not present.
        """
        match = self.MATCHER.match(line)
        if not match:
            raise ParseError(lineno, 'Syntax error: %s' % line)

        reason, raw_tags, test, raw_results, trailing_comments = match.groups()

        # TODO(rmhasan): Find a better regex to capture the reasons. The '*' in
        # the reasons regex only allows us to get the last bug. We need to write
        # the below code to get the full list of reasons.
        if reason:
            reason = reason.strip()
            index = line.find(reason)
            reason = line[:index] + reason
        return reason, raw_tags, test, raw_results, trailing_comments

    def _validate_expectation_structure(self, lineno, test, tags):
        """Helper function to validate aspects of an expectation being parsed.

        Specifically, checks for:
            * Correct use of wildcard characters
            * All tags used are known
            * Only one tag from each tag set is used
        """
        tag_set_ids = set()
        for i in range(len(test)-1):
            if test[i] == '*' and ((i > 0 and test[i-1] != '\\') or i == 0):
                raise ParseError(lineno,
                    'Invalid glob, \'*\' can only be at the end of the pattern')

        for t in tags:
            if not t in  self._tag_to_tag_set:
                raise ParseError(lineno, 'Unknown tag "%s"' % t)
            else:
                tag_set_ids.add(self._tag_to_tag_set[t])

        if len(tag_set_ids) != len(tags):
            error_msg = ('The tag group contains tags that are '
                         'part of the same tag set')
            tags_by_tag_set_id = defaultdict(list)
            for t in tags:
              tags_by_tag_set_id[self._tag_to_tag_set[t]].append(t)
            for tag_intersection in tags_by_tag_set_id.values():
                error_msg += ('\n  - Tags %s are part of the same tag set' %
                              _group_to_string(sorted(tag_intersection)))
            raise ParseError(lineno, error_msg)

    def _parse_and_validate_raw_results(self, lineno, raw_results):
        """Helper function to validate and parse raw results into known values.

        Returns:
            A tuple (results, retry_on_failure, is_slow_test). |results| is a
            list of parsed results. |retry_on_failure| is a boolean denoting
            whether the test should be retried on failure or not. |is_slow_test|
            is a boolean denoting whether the test should be considered slow or
            not.
        """
        results = []
        retry_on_failure = False
        is_slow_test = False
        for r in raw_results.split():
            if r not in self._allowed_results:
                raise ParseError(lineno, 'Unknown result type "%s"' % r)
            try:
                # The test expectations may contain expected results and
                # the RetryOnFailure tag
                if r in  _EXPECTATION_MAP:
                    results.append(_EXPECTATION_MAP[r])
                elif r == _RETRY_ON_FAILURE_TAG:
                    retry_on_failure = True
                elif r == _SLOW_TAG:
                    is_slow_test = True
                else:
                    raise KeyError
            except KeyError:
                raise ParseError(lineno, 'Unknown result type "%s"' % r)
        return results, retry_on_failure, is_slow_test

class TestExpectations(object):

    def __init__(self, tags=None, ignored_tags=None):
        self.tag_sets = set()
        self.ignored_tags = set(ignored_tags or [])
        self.set_tags(tags or [])
        # Expectations may either refer to individual tests, or globs of
        # tests. Each test (or glob) may have multiple sets of tags and
        # expected results, so we store these in dicts ordered by the string
        # for ease of retrieve. glob_exps use an OrderedDict rather than
        # a regular dict for reasons given below.
        self.individual_exps = OrderedDict()
        self.glob_exps = OrderedDict()
        self._tags_conflict = _default_tags_conflict
        self._conflict_resolution = ConflictResolutionTypes.UNION

    def set_tags(self, tags, raise_ex_for_bad_tags=False):
        self.validate_condition_tags(tags, raise_ex_for_bad_tags)
        self._tags = [tag.lower() for tag in tags]

    def add_tags(self, new_tags, raise_ex_for_bad_tags=False):
        self.validate_condition_tags(new_tags, raise_ex_for_bad_tags)
        self._tags = list(
            set(self._tags) | set([tag.lower() for tag in new_tags]))

    @property
    def tags(self):
        return self._tags[:]

    def validate_condition_tags(self, tags, raise_ex_for_bad_tags):
        # This function will be used to validate if each tag in the tags list
        # is declared in a test expectations file. This validation will make
        # sure that the tags written in the test expectations files match tags
        # that are generated by the test runner.
        def _pluralize_unknown(missing):
            if len(missing) > 1:
                return ('s %s and %s are' % (', '.join(missing[:-1]),
                                             missing[-1]),
                        'have', 's are')
            else:
                return (' %s is' % missing[0], 'has', ' is')
        tags = set(t.lower() for t in tags)
        unknown_tags = set()
        if self.tag_sets:
            known_and_ignored_tags = set().union(
                *self.tag_sets).union(self.ignored_tags)
            unknown_tags = tags - known_and_ignored_tags
        if unknown_tags:
            msg = (
                'Tag%s not declared in the expectations file and %s not been '
                'explicitly ignored by the test. There may have been a typo in '
                'the expectations file. Please make sure the aforementioned '
                'tag%s declared at the top of the expectations file.' %
                _pluralize_unknown(sorted(unknown_tags)))
            if raise_ex_for_bad_tags:
                raise ValueError(msg)
            else:
                logging.warning(msg)

    def parse_tagged_list(self, raw_data, file_name='',
                          tags_conflict=_default_tags_conflict,
                          conflict_resolution=ConflictResolutionTypes.UNION):
        ret = 0
        self.file_name = file_name
        self._conflict_resolution = conflict_resolution
        try:
            parser = TaggedTestListParser(raw_data, conflict_resolution)
        except ParseError as e:
            return 1, str(e)
        # If we have parsed another tagged list before, ensure that the tag sets
        # are the same in order to prevent any ambiguity about which set a tag
        # belongs to.
        if self.tag_sets:
            if not self.tag_sets == parser.tag_sets:
                raise RuntimeError(
                    'Existing tag sets %s do not match incoming sets %s' % (
                        sorted(self.tag_sets), sorted(parser.tag_sets)))
        else:
            self.tag_sets = parser.tag_sets
        self._tags_conflict = tags_conflict
        # Conflict resolution tag in raw data will take precedence
        self._conflict_resolution = parser.conflict_resolution
        # TODO(crbug.com/83560) - Add support for multiple policies
        # for supporting multiple matching lines, e.g., allow/union,
        # reject, etc. Right now, you effectively just get a union.
        glob_exps = []
        for exp in parser.expectations:
            if exp.is_glob:
                glob_exps.append(exp)
            else:
                self.individual_exps.setdefault(exp.test, []).append(exp)

        # Each glob may also have multiple matching lines. By ordering the
        # globs by decreasing length, this allows us to find the most
        # specific glob by a simple linear search in expected_results_for().
        glob_exps.sort(key=lambda exp: len(exp.test), reverse=True)
        for exp in glob_exps:
            self.glob_exps.setdefault(exp.test, []).append(exp)

        errors = ''
        if not parser.conflicts_allowed:
            errors = self.check_test_expectations_patterns_for_conflicts()
            ret = 1 if errors else 0
        return ret, errors

    def merge_test_expectations(self, other):
        # Merges another TestExpectation instance into this instance.
        # It will merge the other instance's and this instance's
        # individual_exps and glob_exps dictionaries.
        self.add_tags(other.tags)
        for pattern, exps in other.individual_exps.items():
            self.individual_exps.setdefault(pattern, []).extend(exps)
        for pattern, exps in other.glob_exps.items():
            self.glob_exps.setdefault(pattern, []).extend(exps)
        # resort the glob patterns by length in self.glob_exps ordered
        # dictionary
        glob_exps = self.glob_exps
        self.glob_exps = OrderedDict()
        for pattern, exps in sorted(
              glob_exps.items(), key=lambda item: len(item[0]), reverse=True):
            self.glob_exps[pattern] = exps

    def expectations_for(self, test):
        # Returns an Expectation.
        #
        # A given test may have multiple expectations, each with different
        # sets of tags that apply and different expected results, e.g.:
        #
        #  [ Mac ] TestFoo.test_bar [ Skip ]
        #  [ Debug Win ] TestFoo.test_bar [ Pass Failure ]
        #
        # To determine the expected results for a test, we have to loop over
        # all of the failures matching a test, find the ones whose tags are
        # a subset of the ones in effect, and  return the union of all of the
        # results. For example, if the runner is running with {Debug, Mac, Mac10.12}
        # then lines with no tags, {Mac}, or {Debug, Mac} would all match, but
        # {Debug, Win} would not. We also have to set the should_retry_on_failure
        # boolean variable to True if any of the expectations have the
        # should_retry_on_failure flag set to true
        #
        # The longest matching test string (name or glob) has priority.

        # Ensure that the given test name is in the same decoded format that
        # is used internally so that %20 is handled properly.
        test = uri_decode_spaces(test)
        self._results = set()
        self._reasons = set()
        self._exp_tags = set()
        self._should_retry_on_failure = False
        self._is_slow_test = False
        self._trailing_comments = str()

        def _update_expected_results(exp):
            if exp.tags.issubset(self._tags):
                if exp.conflict_resolution == ConflictResolutionTypes.UNION:
                    if not exp.is_default_pass:
                        self._results.update(exp.results)
                    self._should_retry_on_failure |= exp.should_retry_on_failure
                    self._is_slow_test |= exp.is_slow_test
                    self._exp_tags.update(exp.tags)
                    if exp.trailing_comments:
                        self._trailing_comments += exp.trailing_comments + '\n'
                    if exp.reason:
                        self._reasons.update([exp.reason])
                else:
                    self._results = set(exp.results)
                    self._should_retry_on_failure = exp.should_retry_on_failure
                    self._is_slow_test = exp.is_slow_test
                    self._exp_tags = set(exp.tags)
                    self._trailing_comments = exp.trailing_comments
                    if exp.reason:
                        self._reasons = {exp.reason}

        # First, check for an exact match on the test name.
        for exp in self.individual_exps.get(test, []):
            _update_expected_results(exp)

        if self._results or self._is_slow_test or self._should_retry_on_failure:
            return Expectation(
                    test=test, results=self._results, tags=self._exp_tags,
                    retry_on_failure=self._should_retry_on_failure,
                    conflict_resolution=self._conflict_resolution,
                    is_slow_test=self._is_slow_test, reason=' '.join(sorted(self._reasons)),
                    trailing_comments=self._trailing_comments)

        # If we didn't find an exact match, check for matching globs. Match by
        # the most specific (i.e., longest) glob first. Because self.globs_exps
        # is ordered by length, this is a simple linear search
        for glob, exps in self.glob_exps.items():
            glob = glob[:-1]
            if test.startswith(glob):
                for exp in exps:
                    _update_expected_results(exp)
                # if *any* of the exps matched, results will be non-empty,
                # and we're done. If not, keep looking through ever-shorter
                # globs.
                if self._results or self._is_slow_test or self._should_retry_on_failure:
                    return Expectation(
                            test=test, results=self._results, tags=self._exp_tags,
                            retry_on_failure=self._should_retry_on_failure,
                            conflict_resolution=self._conflict_resolution,
                            is_slow_test=self._is_slow_test, reason=' '.join(sorted(self._reasons)),
                            trailing_comments=self._trailing_comments)

        # Nothing matched, so by default, the test is expected to pass.
        return Expectation(test=test)

    def tag_sets_conflict(self, s1, s2):
        # Tag sets s1 and s2 have no conflict when there exists a tag in s1
        # and tag in s2 that are from the same tag declaration set and do not
        # conflict with each other.
        for tag_set in self.tag_sets:
            for t1, t2 in itertools.product(s1, s2):
                if (t1 in tag_set and t2 in tag_set and
                    self._tags_conflict(t1, t2)):
                    return False
        return True

    def check_test_expectations_patterns_for_conflicts(self):
        # This function makes sure that any test expectations that have the same
        # pattern do not conflict with each other. Test expectations conflict
        # if their tag sets do not have conflicting tags. Tags conflict when
        # they belong to the same tag declaration set. For example an
        # expectations file may have a tag declaration set for operating systems
        # which might look like [ win linux]. A test expectation that has the
        # linux tag will not conflict with an expectation that has the win tag.
        error_msg = ''
        patterns_to_exps = dict(self.individual_exps)
        patterns_to_exps.update(self.glob_exps)
        for pattern, exps in patterns_to_exps.items():
            conflicts_exist = False
            for e1, e2 in itertools.combinations(exps, 2):
                if self.tag_sets_conflict(e1.tags, e2.tags):
                    if not conflicts_exist:
                        error_msg += (
                            '\nFound conflicts for pattern %s%s:\n' %
                            (pattern,
                             (' in %s' %
                              self.file_name if self.file_name else '')))
                    conflicts_exist = True
                    error_msg += ('  line %d conflicts with line %d\n' %
                                  (e1.lineno, e2.lineno))
        return error_msg

    def check_for_broken_expectations(self, test_names):
        # It returns a list expectations that do not apply to any test names in
        # the test_names list.
        #
        # args:
        # test_names: list of test names that are used to find test expectations
        # that do not apply to any of test names in the list.
        broken_exps = []
        # Apply the same temporary encoding we do when ingesting/comparing
        # expectations.
        test_names = [uri_decode_spaces(tn) for tn in test_names]
        test_names = set(test_names)
        for pattern, exps in self.individual_exps.items():
            if pattern not in test_names:
                broken_exps.extend(exps)

        # look for broken glob expectations
        # first create a trie of test names
        trie = {}
        broken_glob_exps = []
        for test in test_names:
            _trie = trie.setdefault(test[0], {})
            for l in test[1:]:
                _trie = _trie.setdefault(l, {})
            _trie.setdefault('\0', {})

        # look for globs that do not match any test names and append their
        # expectations to glob_broken_exps
        for pattern, exps in self.glob_exps.items():
            _trie = trie
            for i, l in enumerate(pattern):
                if l == '*' and i == len(pattern) - 1:
                    break
                if l not in _trie:
                    broken_glob_exps.extend(exps)
                    break
                _trie = _trie[l]
        return broken_exps + broken_glob_exps

def uri_encode_spaces(s):
  s = s.replace('%', '%25')
  s = s.replace(' ', '%20')
  return s

def uri_decode_spaces(s):
  s = s.replace('%20', ' ')
  s = s.replace('%25', '%')
  return s


from Bio import SeqIO

seq_count = 50
seq_length = 5000

def main():
    fasta_sequences = SeqIO.parse(open("felCat8.fa"), 'fasta')
    i = 0

    with open('UCSC_Cat.txt', 'w') as f:
        for fasta in fasta_sequences:
            if i == seq_count:
                break
            f.write(str(fasta.seq).replace("N", "")[:seq_length].upper())
            f.write('\n')
            i += 1

    f.close()

if __name__ == '__main__':
    main()


contador_letras = lambda lista: [len(x) for x in lista]

lista_palavras = ['cachorro', 'gato', 'elefante']
print(contador_letras(lista_palavras))

soma = lambda a, b: a + b
subtracao = lambda a, b: a - b
multiplicacao = lambda a, b: a * b
divisao = lambda a, b: a / b
print(soma(10, 5))
print(subtracao(10, 5))
print(multiplicacao(10, 5))
print(divisao(10, 5))

caluladora = {
    'soma': lambda a, b: a + b,
    'subtracao': lambda a, b: a - b,
    'multiplicacao': lambda a, b: a * b,
    'divisao': lambda a, b: a / b
}

print(type(caluladora))
soma = caluladora['soma']
subtracao = caluladora['subtracao']
multiplicacao = caluladora['multiplicacao']
divisao = caluladora['divisao']
print('A soma entre A e B: {}'.format(soma(10, 5)))
print('A subtração entre A e B: {}'.format(subtracao(10, 5)))
print('A multiplicação entre A e B: {}'.format(multiplicacao(10, 5)))
print('A divisão entre A e B: {}'.format(divisao(10, 5)))


row,column=input().split()
mine_number = int(input())
mine_coordinate=[]
for mine in range(mine_number):
   mine_coordinate.append(list(map(int,input().split())))
mine_coordinate.sort()
row=int(row)
column=int(column)
map = []
for x in range(row):
    map.append([])
    for y in range(column):
        map[x].append(0)
for mine in mine_coordinate:
    map[mine[0]-1][mine[1]-1]='*'
###################################
for i in range(row):
    for j in range(column):
        if map[i][j]=='*':
            if i-1>=0 and j-1>=0 and map[i-1][j-1]!='*':
                map[i-1][j-1]+=1
            if i-1>=0 and map[i-1][j]!='*':
                map[i-1][j]+=1
            if i-1>=0 and j+1<=column-1 and map[i-1][j+1]!='*':
                map[i-1][j+1]+=1
            if j-1>=0 and map[i][j-1]!='*':
                map[i][j-1]+=1
            if j+1<=column-1 and map[i][j+1]!='*':
                map[i][j+1]+=1
            if i+1<=row-1 and j-1>=0 and map[i+1][j-1]!='*':
                map[i+1][j-1]+=1
            if i+1<=row-1 and map[i+1][j]!='*':
                map[i+1][j]+=1
            if i+1<=row-1 and j+1<=column-1 and map[i+1][j+1]!='*':
                map[i+1][j+1]+=1
for k in range(row):
    for t in range(column):
        print(map[k][t],end=' ')
    print("")


# N = 500
# N^3 가능
from sys import stdin

N, M = map(int, stdin.readline().split())
MAX = 10000

realtions = [ [ MAX  for _ in range(N+1)] for _ in range(N+1) ]
degree = [ 0 for _ in range(N+1)]

for m in range(M):
    f, t = map(int, stdin.readline().split())

    realtions[f][t] = 1

for k in range(1, N+1):
    for j in range(1, N+1):
        for i in range(1, N+1):
            if realtions[i][j] > realtions[i][k] + realtions[k][j]:
                realtions[i][j] = realtions[i][k] + realtions[k][j]

for i in range(1, N+1):
    for j in range(1, N+1):
        if realtions[i][j] != MAX:
            degree[i] += 1
            degree[j] += 1
result = 0
for i in range(1, N+1):
    if degree[i] == N-1:
        result += 1

print(result)


from unittest import mock, TestCase

from unleash_client import features

class TestFactory(TestCase):
    def test_simple_case(self):
        strategies = {'Foo': mock.Mock(return_value='R')}
        feature = {'strategies': [{'name': 'Foo', 'parameters': {'x': 0}}]}

        result = features.feature_gates(strategies, feature)

        assert result == ['R']
        assert strategies['Foo'].called_once_with(x=0)

    def test_two_strategies(self):
        strategies = {
            'Foo': mock.Mock(return_value='F'),
            'Bar': mock.Mock(return_value='B'),
        }
        feature = {'strategies': [
            {'name': 'Foo', 'parameters': {'x': 0}},
            {'name': 'Bar', 'parameters': {'y': 1}},
        ]}

        result = features.feature_gates(strategies, feature)

        assert result == ['F', 'B']
        assert strategies['Foo'].called_once_with(x=0)
        assert strategies['Bar'].called_once_with(y=1)

    def test_unknown_strategy(self):
        strategies = {}
        feature = {'strategies': [{'name': 'absent', 'parameters': {'z': 9}}]}

        with mock.patch('unleash_client.features.log') as log:
            result = features.feature_gates(strategies, feature)

        assert len(result) == 1
        assert callable(result[0])
        assert not result[0](basically_anything_here='foo')
        assert log.warning.called

class TestFeature(TestCase):
    def test_happy_path(self):
        strategies = {'Foo': mock.Mock(return_value=lambda z: z)}
        feature_def = {
            'enabled': True,
            'strategies': [{'name': 'Foo', 'parameters': {'x': 0}}],
        }

        toggle = features.Feature(strategies, feature_def)

        assert isinstance(toggle, features.Feature)
        assert toggle({'z': True})
        assert not toggle({'z': False})
        assert toggle.choices == {True: 1, False: 1}

    def test_empty_strategy_list(self):
        strategies = {'Foo': mock.Mock(return_value=lambda z: z)}
        feature_def = {
            'enabled': True,
            'strategies': [],
        }

        toggle = features.Feature(strategies, feature_def)

        assert isinstance(toggle, features.Feature)
        assert not toggle({'z': True})
        assert not toggle({'z': False})
        assert toggle.choices == {True: 0, False: 2}

    def test_disable(self):
        strategies = {'Foo': mock.Mock(return_value=lambda z: z)}
        feature_def = {
            'enabled': False,
            'strategies': [{'name': 'Foo', 'parameters': {'x': 0}}],
        }

        toggle = features.Feature(strategies, feature_def)

        assert not toggle({'z': True})
        assert not toggle({'z': False})
        assert toggle.choices == {True: 0, False: 2}


class Date:
    def __init__(self, day_month_year):
        self.day_month_year = str(day_month_year)

    @classmethod
    def printing(cls, day_month_year):
        date_list = []
        for i in day_month_year.split():
            if i != '-':
                date_list.append(i)

        return f'{(date_list[0])}, {(date_list[1])}, {(date_list[2])}'

    @staticmethod
    def valid(day, month, year):
        if 0 < day < 32 and 0 < month < 13:
            if month == 2 and 0 < day < 29 and year % 4 != 0:
                month = str(month)
                return f'{day}.{month.zfill(2)}.{year}'
            elif year % 4 == 0 and month == 2 and 0 < day < 30 and year > 0:
                month = str(month)
                return f'{day}.{month.zfill(2)}.{year}'
            elif month != 2:
                month = str(month)
                return f'{day}.{month.zfill(2)}.{year}'

            else:
                return f'Не верно введена дата'

        else:
            return f'Не верно введена дата'

date = Date('12 - 2 - 1996')

print(date.printing('12 - 2 - 1996'))
print(Date.valid(28, 12, 1996))
print(Date.valid(29, 2, 1997))


import numpy as np
import matplotlib.pyplot as plt

from tqdm import tqdm

from bandit_strategies import Greedy, EpsilonGreedy, DecayingEpsilonGreedy, OptimisticInitialValues, UCB1, ThomsonSampling

N_TRIALS = int(1e5)
BANDIT_PROBABILITIES = [.2, .5, .75]

def run_experiment(strat):
    '''
    :param strat: Strategy; Instance of class implementing strategy API
    '''
    print(strat)

    # init exp recorders
    rewards = []
    explored = []

    # init bandits
    bandits = [strat.init_bandit(p) for p in BANDIT_PROBABILITIES]

    # main loop
    for trail_n in tqdm(range(N_TRIALS)):
        chosen_bandit, exploring = strat.choose(bandits, trail_n)

        # update recorders
        rewards.append(chosen_bandit.pull())
        explored.append(exploring)

    #report
    print(f'TOTAL REWARD: {sum(rewards)}')
    print(f'WIN RATE: {100 * sum(rewards)/N_TRIALS:.4f}%')
    print(f'TOTAL TIMES EXPLORING: {sum(explored)}')
    print(f'BANDIT ESTIMATES: {[b.p_estimate for b in bandits]}')

    #plot
    cum_rewards = np.cumsum(rewards)
    win_rate = cum_rewards / (np.arange(N_TRIALS) + 1)
    plt.plot(win_rate, label=str(strat))
    plt.plot(np.ones(N_TRIALS) * np.max(BANDIT_PROBABILITIES))
    plt.xscale('log')
    plt.ylim(0, 1)
    # plt.show()

if __name__ == '__main__':
    plt.figure()

    run_experiment(EpsilonGreedy(epsilon=.1))
    run_experiment(DecayingEpsilonGreedy(initial_epsilon=.1, decay_factor=.0001))
    run_experiment(OptimisticInitialValues(initial_value=5.))
    run_experiment(UCB1(
        trial_n_offset=len(BANDIT_PROBABILITIES) # equivalent to running each bandit once before going into main loop
        ))
    run_experiment(ThomsonSampling())

    plt.legend()
    plt.show()


# packages
import numpy as np
from tensorflow.keras import layers
import tensorflow as tf
import pandas as pd
import plot_utils

class MLP:
    def __init__(self, train_X, train_Y, test_X, test_Y, layer_sizes, loss='mean_squared_error',
                 optimizer='adam', regularizer = 'l2', validation_split=0.2, epochs=10000, batch_size=1,
                 patience=5, lookback=5, plot=True):
        self.loss = loss
        self.optimizer = optimizer
        self.regularizer = regularizer
        self.validation_split = validation_split
        self.epochs = epochs
        self.batch_size = batch_size
        self.patience = patience
        self.lookback = lookback

        self.old_shape = train_X.shape
        self.train_X = self.adjust_data(train_X)
        self.train_Y = train_Y
        self.test_X = self.adjust_data(test_X)
        self.test_Y = test_Y

        self.plot = plot
        self.model = self.train_model(layer_sizes)

    def train_model(self, layer_sizes):
        callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=self.patience)
        model = tf.keras.Sequential()

        for size in layer_sizes:
            model.add(layers.Dense(size))

        model.add(layers.Dense(12, kernel_regularizer=self.regularizer))
        model.compile(loss=self.loss, optimizer=self.optimizer)

        history = model.fit(self.train_X, self.train_Y,
                            validation_split=self.validation_split,
                            epochs=self.epochs,
                            batch_size=self.batch_size,
                            callbacks=[callback],
                            shuffle=False,
                            verbose=2)
        print("MSE on test data: ", model.evaluate(self.test_X, self.test_Y))

        if self.plot:
            plot_utils.plot_loss(history, True, 'MLP')

        return model

    def make_new_predictions(self, horizon):
        input = self.test_X[0].reshape(1, self.test_X[0].shape[0])
        preds = []
        for _ in range(horizon):
            pred = self.model.predict(input)
            preds.append(pred)
            input = np.concatenate((input[:, self.old_shape[2]:].reshape(1, input.shape[0] - self.old_shape[2]), pred),
                                   axis=1)

        if self.plot:
            plot_utils.plot_trajectories(preds, self.test_Y[0:horizon], horizon, True, 'MLP')

    def adjust_data(self, data):
        first = np.reshape(data[0], (1, self.lookback*data[0].shape[1]))

        for i in range(1, data.shape[0]):
            first = np.concatenate((first, np.reshape(data[i], (1, self.lookback*data[i].shape[1]))))

        return first