# -*- coding: utf-8 -*-
"""mapping

Container to collect, store, and process universe inputs provided in a map.

First the different channels are created and then the map with all these
channels must be provided.

Created on Mon May 01 13:00:00 2023 @author: Dan Kotlyar
Last updated on Mon Sep 04 06:15:00 2023 @author: Dan Kotlyar

email: dan.kotlyar@me.gatech.edu

List changes / additions:
--------------------------
"Concise description" - MM/DD/YYY - Name initials
Validate - 05/02/2023 - DK
_reshapeTo1D, _reshapeTo3D - 09/04/2023 - DK

"""

import copy

import numpy as np

from xsInterface.errors.checkerrors import _isstr, _inlist,\
    _is1darray, _inrange, _isequallength, _islist, _compare2lists

ALLOWED_MANIPULATION = ['multiply', 'divide']

class Map:
    """A container to store the input data for all channels

    The data for all the radial channels is included in this container.
    In addition, the radial distribution (map) of the channels is provided.
    Each channel contains the information about the axial distribution.

    Parameters
    ----------
    radmap : 2D list of str
        description of the channels radial distribution/map
        e.g.
        ...   `1` `1` `1`
        ... `2` `3` `3` `2`
        ...   `1` `1` `1`
    idxBounds : 2D list of int
        indices of the start and end position for each row
        e.g. [[-1, 1], [-2, 1], [-1, 1]]
        In the above example [-1, 1] represents the components that correspond
        to columns -1, 0, and 1. For this specific row, three entries will
        be expected in the ``radmap``.
        The entries do not have to start from negative numbers. An alternative
        for the above example is: [[0, 2], [-1, 2], [0, 2]] or
        [[1, 3], [0, 3], [1, 3]]

    Attributes
    ----------
    radmap : 2D list of str
        square matrix with radial distribution of radial channels
        ``None`` represent the lack of a channel.
    chIds : set of strings
        Identifiers/names of all the unique channels.
        Following the above example, the unique channels identifiers are `1`
        and `2`.

    Methods
    -------
    AddChannel : add a new channel object
    GetChannel : obtain a specific channel object
    Validate : post validation to check that all channels are defined properly

    Raises
    ------
    TypeError
        If ``radmap`` or ``idxBounds`` is not a list.
    ValueError
        If the number of entries for each row in ``radmap`` does not allign
        with the starting and ending indices in ``idxBounds``.

    Notes
    -----
    The values passed to ``radmap`` should correspond to exisiting channels
    defined using the ``Channel`` container.

    Examples
    --------
    >>> core = Map([['A'], ['B', 'A', 'B'], ['C']],
    ... [[0, 0], [-1, 1], [0, 0]])

    """

    def __init__(self, radmap, idxBounds):

        # check potential errors
        _islist(radmap, "channels radial map")
        _islist(idxBounds, "bounds of left and right indices")

        if len(radmap) != len(idxBounds):
            raise ValueError("number of rows in radmap {} and idxBounds {} "
                             "must be equal".format(radmap, idxBounds))

        bounds = np.asarray(idxBounds)  # convert from a list
        # Outermost left and right bounds
        maxBounds = bounds.min(), bounds.max()

        # Init a square radial map:
        nrows = len(radmap)  # number of rows
        ncols = maxBounds[1] - maxBounds[0] + 1  # number of columns
        self.radmap = [[None for x in range(ncols)] for y in range(nrows)]
        self.idxmap = copy.deepcopy(self.radmap)

        # the left and right bounds must align to the number
        # of values in the corresponding row of the radial map
        for idx, rowVals in enumerate(radmap):
            rowBounds = idxBounds[idx]
            if len(rowBounds) != 2:
                raise ValueError("Only two values (left and right bounds) "
                                 "are expected and not {}".format(rowBounds))

            # check that bounds of each row are within max. bounds
            _inrange(rowBounds[0], "left row bound", maxBounds)
            _inrange(rowBounds[1], "right row bound", maxBounds)

            if rowBounds[1] < rowBounds[0]:
                raise ValueError("Right bound {} cannot be lower than left "
                                 "bound {}".format(rowBounds[1], rowBounds[0]))

            if rowBounds[1] - rowBounds[0] + 1 != len(rowVals):
                raise ValueError("The left/right bounds {} do not match the "
                                 "number of entries {}"
                                 .format(rowBounds, rowVals))

            self.radmap[idx][rowBounds[0] - maxBounds[0]:
                             rowBounds[1] - maxBounds[0] + 1] = rowVals
        self.idxBounds = idxBounds
        # Init to empty dictionary for storing data of a specific channel
        self.channels = {}
        # Identify all the unique channels types
        self.chIds = [] #set() +DK
        for row in radmap:
            self.chIds = self.chIds + row  # +DK
            # self.chIds.update(row)
        self.chIds = list(np.unique(self.chIds))

        self.linkChannelsInMap = {}
        self.orderedChIds = []
        c = 0  # the first channel is the zero channel
        for irow, row in enumerate(radmap):
            for jcol, col in enumerate(row):
                self.idxmap[irow][jcol] = c
                # map the channels to their positions
                chId = radmap[irow][jcol]  # get chId in this location
                self.orderedChIds.append(chId)
                if chId in self.linkChannelsInMap:
                    self.linkChannelsInMap[chId] =\
                        self.linkChannelsInMap[chId] + [c]
                else:
                    self.linkChannelsInMap[chId] = [c]
                c += 1

        # core operational states for the results (not used here directly)
        self.corestates = {}
        # core values for different attributes (not used here directly)
        self.corevalues = {}

    def Channel(self, chId, universes, volumes=None):
        """Add the name, universes and volumes for a specific channel

        Parameters
        ----------
        chId : str
            identifier/name of the channel
        universes : list of strings
            universe for each of the layers in this channels
        volumes : list or array
            volume for each of the layers in this channels

        Raises
        ------
        TypeError
            If the ``chId`` is not a string.
            If ``universes`` not a list.
        KeyError
            If the channel already exists.
            If the channel Id does not appear in ``radmap``.

        Examples
        --------
        >>> core.Channel("A", ['u0', 'u0', 'u0', 'u0', 'u0'],
                         [1, 1, 1, 1, 1])

        """

        _isstr(chId, "Channel Id")
        if chId not in self.chIds:
            raise ValueError("ChId {} does not exist in {}"
                             .format(chId, self.chIds))

        # check that universes are provided as a list
        _islist(universes, "universes")

        # check that list is not empty
        nvals = len(universes)
        if nvals == 0:
            raise ValueError("universes cannot be empty.")

        # if volume is not provided set to unity volume
        if volumes is None:
            volumes = np.ones(nvals)

        # convert to numpy array (if provided as a list)
        volumes = np.array(volumes)

        # arrays' dimensions
        _is1darray(volumes, "volumes")

        # number of values in ``volumes``
        _isequallength(volumes, nvals, "volumes")

        # check if the channel was already defined
        if chId in self.channels:
            raise ValueError("Channel <{}> already exist.".format(chId))

        # assign values to the specific channel
        self.channels[chId] = {"nlayers": nvals,
                               "layers": universes,
                               "volumes": volumes}

    def Copy(self, existId, newId):
        """Copy an already existing universe for a new universe Id

        Parameters
        ----------
        existId : str
            identifier/name of the channel
        newId : list of strings
            universe for each of the layers in this channels

        Raises
        ------
        TypeError
            If the ``existId`` or ``newId`` are not strings.
        KeyError
            If the ``existId`` does not exist.
            If the ``newId`` does not already exist.

        Examples
        --------
        >>> core.Copy("A", "C")

        """

        _isstr(existId, "existId")
        _isstr(newId, "newId")

        if existId not in self.channels:
            raise ValueError("existId {} does not exist in channels {}"
                             .format(existId, self.channels.keys()))

        if newId in self.channels:
            raise ValueError("newId {} already exists in channels {}"
                             .format(newId, self.channels.keys()))

        self.channels[newId] = copy.deepcopy(self[existId])

    def __getitem__(self, pos):
        return self.channels[pos]

    def _NumSpatialNodes(self):
        """count the total number of spatial nodes in the system ch x layers"""
        # this method can only be executed if all the fields already exist

        numChannels = 0
        numNodes = 0
        for chId, chData in self.channels.items():
            chs = self.linkChannelsInMap[chId]
            nchs = len(chs)
            chData['channelsIdx'] = chs   # indices
            chData['nchannels'] = nchs    # number of repetitions
            self.channels[chId] = chData  # update the data
            numChannels += nchs
            numNodes += nchs*chData['nlayers']
        self.numNodes = numNodes
        self.numChannels = numChannels

    def _reshapeTo1D(self, valsIn, normFlag=True, weights=None):
        """reshape the 3D matrix to a 1D vector corresponding to radial map

        Parameters
        ----------
        self : core Map object
            the object includes data about the channels, layers, ng
            the ``self`` object also includes channels Ids
        valsIn : 1-dim list (with 2-dim arrays)
            input vector with the following structure:
            [channel][layer, energy]
            valsIn are provided in the order of the self.chIds
        normFlag : bool
            indicator whether the returned value should be normalized to 1.
        groupWeights : list
            energy group-wise weighting factors for the objective function.
            Should include a value for each enegy group.
            Envisioned to be used for normalization techniques.

        Returns
        -------
        valsOut : 1-dim array
            output vector with the following structure:

             gr1, gr-2, ...    gr1, gr-2, ...  gr1, gr-2, ...    gr1, gr-2, ...
            \______________/ \______________/ \______________/ \______________/
                 layer-0         layer-1                      ...
                \___________________________/  \___________________________/
                         channel-0                      channels-N

        """

        valsChs = [None]*self.numChannels  # total number of channels
        for idxFrom, chId in enumerate(self.chIds):
            # all the parameters that describe the channels
            # nlayers, layers, volumes, channelsIdx, nchannels
            chData = self.channels[chId]

            # 1-dim array with all the values for all layers and group
            # layer0 [g0, g1, ...], layer1 [g0, g1, ...], ...
            valsFrom = valsIn[idxFrom]

            valsFrom = valsFrom.reshape(valsFrom.shape[0]*valsFrom.shape[1])

            # insert the values to the correct channel
            for idxTo in chData['channelsIdx']:
                valsChs[idxTo] = valsFrom

        # produce a 1-dim array [channel][layer][group]
        valsOut = np.concatenate(valsChs)

        # normalization (if needed)
        if weights is not None:
            for ig in range(self.ng):
                valsOut[ig::self.ng] =\
                    weights[ig]*valsOut[ig::self.ng]/valsOut[ig::self.ng].sum()
        elif normFlag:
            valsOut = valsOut / valsOut.sum()

        # final 1-dim numpy array
        return valsOut

    def _reshapeTo3D(self, valsIn):
        """reshape the 1D matrix to a 3D vector corresponding to radial map

        Parameters
        ----------
        self : core Map object
            the object includes data about the channels, layers, ng
            the ``self`` object also includes channels Ids
        valsIn : 1-dim numpy array
            input vector with the following structure:
             gr1, gr-2, ...    gr1, gr-2, ...  gr1, gr-2, ...    gr1, gr-2, ...
            \______________/ \______________/ \______________/ \______________/
                 layer-0         layer-1                      ...
                \___________________________/  \___________________________/
                         channel-0                      channels-N

        Returns
        -------
        valsOut : 3-dim array
            output vector with the following structure:
            [channels][layers][groups]

        """

        numUniqueChannels = len(self.chIds)
        ng = self.ng
        # loop over all the channels in the radial maps
        # even if these have the same chId
        valsInFull = [None]*self.numChannels  # holds the entire distribution
                                              # of channels with repetitive ids
        idx0 = 0
        for idx, chId in enumerate(self.orderedChIds):
            chData = self.channels[chId]  # includes layers, channels, ...

            idx1 = idx0 + chData['nlayers']*ng

            chVals1D = valsIn[idx0:idx1]  # layers and groups for a specific ch
            idx0 = idx1  # update for the next channel

            # reshape the values to 2-m array [layers][groups]
            chVals2D = chVals1D.reshape((chData['nlayers'], ng))

            # insert the layer-group values into the channels values list
            valsInFull[idx] = chVals2D

        # the final 3D list os only defined for the defined unique channels
        valsOut = [None]*numUniqueChannels
        # insert the values to the correct channel
        for idxTo, chId in enumerate(self.chIds):
            chData = self.channels[chId]
            # reset group values for all the layers
            groupVals = np.zeros((chData['nlayers'], ng))
            for idxFrom in chData['channelsIdx']:
                groupVals = groupVals + valsInFull[idxFrom]
            valsOut[idxTo] = groupVals / chData['nchannels']

        # final 3-dim numpy array
        return valsOut

    def Validate(self):
        """post validation to check that all channels were defined

        This function is executed once all the channels were added
        to the ``Map`` container.
        If the number of defined channels is not equal to
        the number of expected channels the ``Validate`` will alert.

        Raises
        ------
        ValueError
            If the number of defined channels does not match the number of
            channels defined in the radial map.

        """

        expChannels = len(self.chIds)
        prdChannels = len(self.channels)
        if prdChannels != expChannels:
            raise ValueError("{} channels exist in the radmap, but {} "
                             "channels were defined "
                             .format(expChannels, prdChannels))

        # Check for potential errors
        chMap = list(self.chIds)
        chChnls = list(self.channels.keys())
        _compare2lists(chMap, chChnls, "Channel names in set <map>",
                       "Channel names in set <channels>")

    # def Values(self, attr, chIds=None, volManip=None, **kwargs):
    #     """Obtain the values of a single attribute for all the channels

    #     This method returns a 2-dim list for a specific attribute across
    #     all channels and layers.

    #     Parameters
    #     ----------
    #     attr : string
    #         name of the attribute
    #     chIds : list of string
    #         list with all the channel names. If None, the results for all the
    #         channels is provided.
    #     volManip : string
    #         volume manipulation ['multiply', 'divide']. Default is None.
    #     kwargs : named arguments
    #         keys represent the state name and value represent the values.
    #         The filtering of data is performed according to kwargs.

    #     Examples
    #     --------
    #     >>> universes.Values("u0", attr="inf_nsf", fuel=900)
    #     ... TBC ....

    #     """

    #     if chIds is None:
    #         chIds = self.core.chIds

    #     nchannels = len(chIds)  # number of channels
    #     results = [None]*nchannels  # results will be stored here

    #     # check that the states for all the channels are provided
    #     for key, value in kwargs.items():
    #         nvals = len(value)
    #         if nvals != nchannels:
    #             raise ValueError(
    #                 "The number of states for <{}> must be of size {} and not "
    #                 "{}".format(key, nchannels, nvals))

    #     for idx, chId in enumerate(chIds):
    #         state = {}  # construct the channel state
    #         for key, value in kwargs.items():
    #             state[key] = value[idx]

    #         # Evaluate the results for a specific channel
    #         results[idx] = self._ChannelValues(chId, attr, volManip, **kwargs)

    #     return results

    # def _ChannelValues(self, chId, attr, volManip=None, **kwargs):
    #     """Obtain the values of a single attribute and corresponding states

    #     This method returns a list vector for all the layers in the channel.
    #     All the state branches must be provided.

    #     Parameters
    #     ----------
    #     chId : string
    #         name of the channel
    #     attr : string
    #         name of the attribute
    #     volManip : string
    #         volume manipulation ['multiply', 'divide']. Default is None.
    #     kwargs : named arguments
    #         keys represent the state name and value represent the values.
    #         The filtering of data is performed according to kwargs.

    #     Examples
    #     --------
    #     >>> core.ValueChannel('ch1', 'infkappa', None,
    #     ...                   history=['nom', 'nom', 'nom', 'nom'],
    #     ...                   time=[0.0, 0.0, 0.0, 0.0],
    #     ...                   fuel=[900, 900, 900, 900],
    #     ...                   boron=[0, 0, 0, 0],
    #     ...                   dens=[700, 700, 700, 700])

    #     """

    #     if volManip is not None:
    #         _inlist(volManip, "Volume manipulation", ALLOWED_MANIPULATION)

    #     if chId not in self.core.chIds:
    #         raise KeyError("chId <{}> does not exist in {}"
    #                        .format(chId, self.core.chIds))

    #     univs = self[chId]['layers']  # all the universes in a specific channel
    #     nlayers = self[chId]['nlayers']  # number of layers
    #     vols = self[chId]['volumes']  # volumes

    #     values = [None]*nlayers

    #     for idx, univ in enumerate(univs):

    #         # loop over all the perturbations to build the current state
    #         state = {}
    #         for key, value in kwargs.items():
    #             if (idx==0) and (len(value) != nlayers):
    #                 raise ValueError(
    #                     "The number of entries for state {} must equal to the"
    #                     " number of universes <{}> in channel <{}>"
    #                     .format(len(value), nlayers, chId))
    #             state[key] = value[idx]

    #         if volManip == 'multiply':
    #             values[idx] = vols[idx]*self.Values(univ, attr, **state)[attr]
    #         elif volManip == 'divide':
    #             values[idx] = self.Values(univ, attr, **state)[attr]/vols[idx]
    #         else:  # None
    #             values[idx] = self.Values(univ, attr, **state)[attr]

    #     return values


import datetime
import mimetypes

from django.db import models

from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic

from mimesis.managers import MediaAssociationManager
from taggit.managers import TaggableManager

class MediaUpload(models.Model):

    title = models.CharField(max_length=150, blank=True)
    description = models.TextField(blank=True)
    media = models.FileField(upload_to="mimesis")
    creator = models.ForeignKey(User)
    created = models.DateTimeField(default=datetime.datetime.now)
    media_type = models.CharField(editable=False, max_length=100)
    media_subtype = models.CharField(editable=False, max_length=100)

    tags = TaggableManager()

    def __unicode__(self):
        return self.title

    @property
    def mime_type(self):
        return "%s/%s" % (self.media_type, self.media_subtype)

    def save(self, *args, **kwargs):
        (mime_type, encoding) = mimetypes.guess_type(self.media.path)
        try:
            mime = mime_type.split("/")
            self.media_type = mime[0]
            self.media_subtype = mime[1]
        except:
            # Mime type unknown, use text/plain
            self.media_type = "text"
            self.media_subtype = "plain"
        super(MediaUpload, self).save()

class MediaAssociation(models.Model):
    """
    A generic association of a MediaUpload object and any other Django model.
    """

    media = models.ForeignKey(MediaUpload)
    description = models.TextField()

    content_type = models.ForeignKey(ContentType)
    object_pk = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey("content_type", "object_pk")

    objects = MediaAssociationManager()


from data import *
from utils.augmentations import SSDAugmentation
from layers.modules import MultiBoxLoss_GMM
from ssd_gmm import build_ssd_gmm
from utils.test_voc import *
from active_learning_loop import *
import os
import sys
import time
import torch
import torch.nn.functional as F
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.nn.init as init
import torch.utils.data as data
import numpy as np
import argparse
import math
import random
from torch.utils.data.sampler import SubsetRandomSampler, SequentialSampler
from subset_sequential_sampler import SubsetSequentialSampler

random.seed(314) # need to change manually
torch.manual_seed(314) # need to change manually

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

parser = argparse.ArgumentParser(
    description='Single Shot MultiBox Detector Training With Pytorch')
train_set = parser.add_mutually_exclusive_group()
parser.add_argument('--dataset', default='VOC', choices=['VOC', 'COCO'],
                    type=str, help='VOC or COCO')
parser.add_argument('--voc_root', default=VOC_ROOT,
                    help='VOC dataset root directory path')
parser.add_argument('--coco_root', default=COCO_ROOT,
                    help='COCO dataset root directory path')
parser.add_argument('--basenet', default='vgg16_reducedfc.pth',
                    help='Pretrained base model')
parser.add_argument('--batch_size', default=32, type=int,
                    help='Batch size for training')
parser.add_argument('--resume', default=None, type=str,
                    help='Checkpoint state_dict file to resume training from')
parser.add_argument('--start_iter', default=0, type=int,
                    help='Resume training at this iter')
parser.add_argument('--num_workers', default=8, type=int,
                    help='Number of workers used in dataloading')
parser.add_argument('--cuda', default=True, type=str2bool,
                    help='Use CUDA to train model')
parser.add_argument('--lr', '--learning-rate', default=1e-3, type=float,
                    help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float,
                    help='Momentum value for optim')
parser.add_argument('--weight_decay', default=5e-4, type=float,
                    help='Weight decay for SGD')
parser.add_argument('--gamma', default=0.1, type=float,
                    help='Gamma update for SGD')
parser.add_argument('--save_folder', default='weights/',
                    help='Directory for saving checkpoint models')
parser.add_argument('--eval_save_folder', default='eval/', type=str,
                    help='File path to save results')
parser.add_argument('--use_cuda', default=True,
                    help='if True use GPU, otherwise use CPU')
parser.add_argument('--id', default=1, type=int,
                    help='the id of the experiment')
parser.add_argument('--top_k', default=5, type=int,
                    help='Further restrict the number of predictions to parse')
parser.add_argument('--confidence_threshold', default=0.01, type=float,
                    help='Detection confidence threshold')
args = parser.parse_args()

device = torch.device("cuda:0" if torch.cuda.is_available() and args.use_cuda else "cpu")

if args.dataset == 'VOC':
    cfg = voc300_active
else:
    cfg = coco300_active

if torch.cuda.is_available():
    if args.cuda:
        torch.set_default_tensor_type('torch.cuda.FloatTensor')
    if not args.cuda:
        print("WARNING: It looks like you have a CUDA device, but aren't " +
              "using CUDA.\nRun with --cuda for optimal training speed.")
        torch.set_default_tensor_type('torch.FloatTensor')
else:
    torch.set_default_tensor_type('torch.FloatTensor')

if not os.path.exists(args.save_folder):
    os.mkdir(args.save_folder)
if not os.path.exists(args.eval_save_folder):
    os.mkdir(args.eval_save_folder)

def create_loaders():
    num_train_images = cfg['num_total_images']
    indices = list(range(num_train_images))
    random.shuffle(indices)
    labeled_set = indices[:cfg['num_initial_labeled_set']]
    unlabeled_set = indices[cfg['num_initial_labeled_set']:]

    if cfg['name'] == 'VOC':
        supervised_dataset = VOCDetection(root=args.voc_root, transform=SSDAugmentation(cfg['min_dim'], MEANS))
        unsupervised_dataset = VOCDetection(args.voc_root, [('2007', 'trainval')],
                                            BaseTransform(300, MEANS),
                                            VOCAnnotationTransform())
    else:
        supervised_dataset = COCODetection(root=args.coco_root, transform=SSDAugmentation(cfg['min_dim'], MEANS))
        unsupervised_dataset = COCODetection(args.coco_root,
                                             transform=BaseTransform(300, MEANS))

    supervised_data_loader = data.DataLoader(supervised_dataset, batch_size=args.batch_size,
                                             num_workers=args.num_workers,
                                             sampler=SubsetRandomSampler(labeled_set),
                                             collate_fn=detection_collate,
                                             pin_memory=True)

    unsupervised_data_loader = data.DataLoader(unsupervised_dataset, batch_size=1,
                                               num_workers=args.num_workers,
                                               sampler=SubsetSequentialSampler(unlabeled_set),
                                               collate_fn=detection_collate,
                                               pin_memory=True)
    return supervised_dataset, supervised_data_loader, unsupervised_dataset, unsupervised_data_loader, indices, labeled_set, unlabeled_set

def change_loaders(
    supervised_dataset,
    unsupervised_dataset,
    labeled_set,
    unlabeled_set,
):
    supervised_data_loader = data.DataLoader(supervised_dataset, batch_size=args.batch_size,
                                             num_workers=args.num_workers,
                                             sampler=SubsetRandomSampler(labeled_set),
                                             collate_fn=detection_collate,
                                             pin_memory=True)

    unsupervised_data_loader = data.DataLoader(unsupervised_dataset, batch_size=1,
                                               num_workers=args.num_workers,
                                               sampler=SubsetSequentialSampler(unlabeled_set),
                                               collate_fn=detection_collate,
                                               pin_memory=True)
    return supervised_data_loader, unsupervised_data_loader

def adjust_learning_rate(
    optimizer,
    gamma,
    step,
):
    """Sets the learning rate to the initial LR decayed by 10 at every
        specified step
    # Adapted from PyTorch Imagenet example:
    # https://github.com/pytorch/examples/blob/master/imagenet/main.py
    """
    lr = args.lr * (gamma ** (step))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def xavier(param):
    init.xavier_uniform(param)

def weights_init(m):
    if isinstance(m, nn.Conv2d):
        xavier(m.weight.data)
        m.bias.data.zero_()

def load_net_optimizer_multi(cfg):
    net = build_ssd_gmm('train', cfg['min_dim'], cfg['num_classes'])

    if args.resume:
        print('Resuming training, loading {}...'.format(args.resume))
        net = nn.DataParallel(net)
        net.load_state_dict(torch.load(args.resume))
    else:
        vgg_weights = torch.load(args.save_folder + args.basenet)
        print('Loading base network...')
        net.vgg.load_state_dict(vgg_weights)
    if not args.resume:
        print('Initializing weights...')
        # initialize newly added layers' weights with xavier method
        net.extras.apply(weights_init)
        net.loc_mu_1.apply(weights_init)
        net.conf_mu_1.apply(weights_init)

    optimizer = optim.SGD(net.parameters(), lr=args.lr,
                          momentum=args.momentum,
                          weight_decay=args.weight_decay)
    if args.cuda:
        net = nn.DataParallel(net)
        net = net.cuda()
    return net, optimizer

def train(
    labeled_set,
    supervised_data_loader,
    indices,
    cfg,
    criterion,
):
    finish_flag = True
    while finish_flag:
        net, optimizer = load_net_optimizer_multi(cfg)
        net = net.train()
        loc_loss = 0
        conf_loss = 0
        supervised_flag = 1
        step_index = 0

        batch_iterator = iter(supervised_data_loader)
        for iteration in range(args.start_iter, cfg['max_iter']):
            # warm-up
            if iteration < 1000:
                lr = args.lr * ((iteration+1)/1000.0)**4
                for param_group in optimizer.param_groups:
                    param_group['lr'] = lr

            if iteration in cfg['lr_steps']:
                step_index += 1
                adjust_learning_rate(optimizer, args.gamma, step_index)
            try:
                images, targets = next(batch_iterator)
            except StopIteration:
                batch_iterator = iter(supervised_data_loader)
                images, targets = next(batch_iterator)

            images = images.to(device)
            targets = [ann.to(device) for ann in targets]

            # forward
            t0 = time.time()
            out = net(images)

            optimizer.zero_grad()
            loss_l, loss_c = criterion(out, targets)
            loss = loss_l + loss_c
            loss.backward()
            optimizer.step()
            t1 = time.time()
            loc_loss += loss_l.data
            conf_loss += loss_c.data

            if (float(loss) > 100) or torch.isinf(loss) or torch.isnan(loss):
                # if the net diverges, go back to point 0 and train from scratch
                break

            if iteration % 10 == 0:
                print('timer: %.4f sec.' % (t1 - t0))
                print('iter ' + repr(iteration) + ' || Loss: %.4f ||' % (loss.data), end=' ')
                print('loss: %.4f , loss_c: %.4f , loss_l: %.4f , lr : %.4f\n' % (
                      loss.data, loss_c.data, loss_l.data, float(optimizer.param_groups[0]['lr'])))

            if iteration != 0 and (iteration + 1) % 10000 == 0:
                print('Saving state, iter:', iteration)
                torch.save(net.state_dict(), 'weights/ssd300_AL_' + cfg['name'] + '_id_' + str(args.id) +
                           '_num_labels_' + str(len(labeled_set)) + '_' + repr(iteration + 1) + '.pth')

            if ((iteration + 1) == cfg['max_iter']):
                finish_flag = False
        else:
            finish_flag = False
    return net

def main():
    if args.cuda:
        cudnn.benchmark = True
    print(args)
    supervised_dataset, supervised_data_loader, unsupervised_dataset, unsupervised_data_loader, indices, labeled_set, unlabeled_set = create_loaders()
    criterion = MultiBoxLoss_GMM(cfg['num_classes'], 0.5, True, 0, True, 3, 0.5, False, args.cuda)
    print(len(labeled_set), len(unlabeled_set))
    net = train(labeled_set, supervised_data_loader, indices, cfg, criterion)

    # # active learning loop
    for i in range(cfg['num_cycles']):
        if cfg['name'] == 'VOC':
            # select the best weight
            list_iter = ['90000', '100000', '110000', '120000']
            list_weights = []
            for loop in list_iter:
                name = 'weights/ssd300_AL_' + cfg['name'] + '_id_' + str(args.id) + '_num_labels_' + str(len(labeled_set)) + '_' + loop + '.pth'
                list_weights.append(str(name))

            list_mean = []
            for loop in list_weights:
                net = build_ssd_gmm('test', cfg['min_dim'], cfg['num_classes'])
                net = nn.DataParallel(net)
                print('loading trained weight {}...'.format(loop))
                net.load_state_dict(torch.load(loop))
                net.eval()
                test_dataset = VOCDetection(args.voc_root, [('2007', 'test')], BaseTransform(300, MEANS), VOCAnnotationTransform())
                mean_ap = test_net(args.eval_save_folder, net, args.cuda,
                                   test_dataset, BaseTransform(300, MEANS),
                                   args.top_k, 300, thresh=args.confidence_threshold)
                list_mean.append(float(mean_ap))
            best_weight = list_weights[list_mean.index(max(list_mean))]

            # active learning
            net = build_ssd_gmm('train', cfg['min_dim'], cfg['num_classes'])
            net = nn.DataParallel(net)
            print('loading best weight {}...'.format(best_weight))
            net.load_state_dict(torch.load(best_weight))

        net.eval()
        batch_iterator = iter(unsupervised_data_loader)
        labeled_set, unlabeled_set = active_learning_cycle(
            batch_iterator,
            labeled_set,
            unlabeled_set,
            net,
            cfg["num_classes"],
            acquisition_budget=cfg['acquisition_budget'],
            num_total_images=cfg['num_total_images'],
        )

        # save the labeled training set list
        f = open("labeled_training_set_" + cfg['name'] + '_' + str(len(labeled_set)) + '_id_' + str(args.id) + ".txt", 'w')
        for i in range(len(labeled_set)):
            f.write(str(labeled_set[i]))
            f.write("\n")
        f.close()

        # change the loaders
        supervised_data_loader, unsupervised_data_loader = change_loaders(supervised_dataset, unsupervised_dataset, labeled_set, unlabeled_set)
        print(len(labeled_set), len(unlabeled_set))

        args.resume = None
        args.start_iter = 0
        net = train(labeled_set, supervised_data_loader, indices, cfg, criterion)

if __name__ == '__main__':
    main()


import traceback
from PySide2.QtWidgets import QMainWindow, QWidget, QMessageBox
from PySide2.QtCore import Signal, QStandardPaths, QFile
from PySide2.QtGui import QShowEvent, QCloseEvent
from typing import Optional
from ui_configwindow import Ui_ConfigWindow
from ui_config_list_item import Ui_ConfigListItem
import json
import os

class ConfigListItem(QWidget):
    removed = Signal(QWidget)

    def __init__(self, parent: Optional[QWidget] = None) -> None:
        super().__init__(parent)
        self.ui = Ui_ConfigListItem()
        self.ui.setupUi(self)
        self.ui.btn_remove.clicked.connect(self.__remove)

    def __remove(self):
        self.removed.emit(self)

class ConfigWindow(QMainWindow):
    closed = Signal(dict)

    def __init__(self, parent: Optional[QWidget] = None) -> None:
        super().__init__(parent)
        self.ui = Ui_ConfigWindow()
        self.ui.setupUi(self)
        self.list_items = []
        self.ui.btn_add.clicked.connect(self.add_config)
        self.cfg_file = QStandardPaths.writableLocation(QStandardPaths.AppDataLocation) + "/config.json"

        version_file = QFile(":/version.txt")
        if version_file.open(QFile.ReadOnly):
            version_txt = bytes(version_file.readLine()).strip().decode()
            self.setWindowTitle("{0} - v{1}".format(self.windowTitle(), version_txt))

    def add_config(self):
        item = ConfigListItem()
        # Default args
        item.ui.txt_args.setPlainText("--dir-cache-time 1m0s\n--vfs-cache-mode full")
        item.removed.connect(self.remove_config)
        self.ui.sa_main.layout().insertWidget(self.ui.sa_main.layout().count() - 1, item)
        self.list_items.append(item)

    def remove_config(self, which: ConfigListItem):
        self.ui.sa_main.layout().removeWidget(which)
        self.list_items.remove(which)
        which.deleteLater()

    def clear_configs(self):
        for cfg_list_item in self.list_items:
            self.ui.sa_main.layout().removeWidget(cfg_list_item)
        self.list_items.clear()

    def show(self, data: dict):
        self.clear_configs()
        for i in range(data["count"]):
            self.add_config()
            obj: ConfigListItem = self.ui.sa_main.layout().itemAt(i).widget()
            obj.ui.txt_remote.setText(data["items"][str(i)]["remote_name"])
            obj.ui.txt_mountpoint.setText(data["items"][str(i)]["mount_point"])
            obj.ui.txt_args.setPlainText(data["items"][str(i)]["mount_args"])
        return super().show()

    def closeEvent(self, event: QCloseEvent):
        try:
            if not os.path.exists(os.path.dirname(self.cfg_file)):
                os.makedirs(os.path.dirname(self.cfg_file))
            with open(self.cfg_file, "w") as f:
                data = {}
                data["count"] = self.ui.sa_main.layout().count() - 1
                data["items"] = {}
                for i in range(data["count"]):
                    data["items"][str(i)] = {}
                    obj: ConfigListItem = self.ui.sa_main.layout().itemAt(i).widget()
                    data["items"][str(i)]["remote_name"] = obj.ui.txt_remote.text()
                    data["items"][str(i)]["mount_point"] = obj.ui.txt_mountpoint.text()
                    data["items"][str(i)]["mount_args"] = obj.ui.txt_args.toPlainText()
                json.dump(data, f)
                self.closed.emit(data)
        except Exception as e:
            traceback.print_exc()
            dialog = QMessageBox(self)
            dialog.setWindowTitle("RcloneDriveManager")
            dialog.setText("Error occurred saving configuration file.")
            dialog.setDetailedText("Exception '{}' occurred.".format(type(e).__name__))
            dialog.setIcon(QMessageBox.Warning)
            dialog.setStandardButtons(QMessageBox.Ok)
            dialog.setDefaultButton(QMessageBox.Ok)
            dialog.exec_()
            self.closed.emit({})
        return super().closeEvent(event)


import streamlit as st
import pandas as pd
import numpy as np
import json
import os
from copy import deepcopy
from PIL import Image
import matplotlib.pyplot as plt
from collections import Counter
from pandas.api.types import is_numeric_dtype
from utils import (
    save_session,
    load_session,
    load_label_json,
    update_label_json,
    update_session_file,
    check_password,
)

HOME_DIRECTORY = ".streamlit"
if os.path.exists(HOME_DIRECTORY) is False:
    os.makedirs(HOME_DIRECTORY)

################# Helper functions ###########################################
@st.cache(allow_output_mutation=True)
def get_state():
    return {}

@st.cache(allow_output_mutation=True)
def get_image_csv(csv_file_path):
    img_df = pd.read_csv(csv_file_path)
    if "img_path" not in img_df:
        st.error("'img_path' doesn't exist in the table")
    img_df["img_name"] = img_df["img_path"].apply(
        lambda x: x.split("/")[-1].split(".")[0]
    )
    if "label" in img_df:
        img_df = img_df.drop(["label"], axis=1)
    return img_df

@st.cache(allow_output_mutation=True)
def dataframe_join_image_label(img_df, label_df):
    img_df = img_df.join(label_df, on="img_path", how="left")
    img_df.loc[img_df["label"].isna(), "label"] = ""
    return img_df

def display_session_setup():
    col1, col2, col3 = st.columns([1, 1, 1])
    username = st.session_state["username"]
    # col1.text_input("Please Enter your user name", key="username")
    col1.write("Current User: " + username)
    session_file = os.path.join(HOME_DIRECTORY, username + "multilabel_image_session")
    if os.path.exists(session_file) and username != "":
        col1.write("You have a saved session: Load or start a new session")
        load_existing_session = col2.button(
            "Load Existing Session", key="load_existing_session"
        )
        col2.text("or")
        diff_session_file = col2.file_uploader(
            "Choose a session file on disk", key="diff_session_file"
        )
    else:
        load_existing_session = False
        col1.write("No session File Found")
        diff_session_file = col2.file_uploader(
            "Choose a session file on disk", key="diff_session_file"
        )

    def start_new_session_fn():
        clean_session()
        # st.session_state["username"] = username
        save_session(session_file, st.session_state)

    col3.write(" ")
    col3.write(" ")
    col3.write(" ")
    col3.write(" ")
    start_new_session = col3.button(
        "Start New Session", on_click=start_new_session_fn, key="start_new_session"
    )
    return username, load_existing_session, diff_session_file, start_new_session

def update_session_profile(session_dict):
    if "label_path" in session_dict:
        st.session_state["label_path"] = session_dict["label_path"]
    if "image_folder" in session_dict:
        st.session_state["image_folder"] = session_dict["image_folder"]
    if "image_csv_file" in session_dict:
        st.session_state["image_csv_file"] = session_dict["image_csv_file"]
    if "username" in session_dict:
        st.session_state["username"] = session_dict["username"]
    if "filter1_values" in session_dict:
        st.session_state["filter1_values"] = tuple(session_dict["filter1_values"])
    if "multiple_labels" in session_dict:
        st.session_state["multiple_labels"] = session_dict["multiple_labels"]

def update_multiple_labels(multiple_labels):
    st.session_state["multiple_labels"] = multiple_labels
    update_session_file()

def clean_session():
    st.session_state["label_path"] = ""
    st.session_state["image_folder"] = ""
    if "filter1_values" in st.session_state:
        del st.session_state["filter1_values"]
    if "multiple_labels" in st.session_state:
        del st.session_state["multiple_labels"]

def label_category(label_str):
    if len(label_str) == 0:
        return "Unlabeled"
    labels = label_str.split(",")
    if len(labels) == 1:
        return "Single Label"
    else:
        return "Multi Label"

def extract_all_labels(label_str_list):
    all_labels = []
    for label_str in label_str_list:
        if len(label_str) > 0:
            labels = label_str.split(",")
            all_labels.extend(labels)
    return all_labels

def extract_multilabels(label_str_list):
    all_labels = []
    for label_str in label_str_list:
        if len(label_str) > 0:
            labels = label_str.split(",")
            if len(labels) > 1:
                all_labels.append(",".join(sorted(labels)))
    return all_labels

def step1_setup_session():
    (
        username,
        load_existing_session,
        diff_session_file,
        start_new_session,
    ) = display_session_setup()
    session_file = os.path.join(HOME_DIRECTORY, username + "multilabel_image_session")
    st.session_state["session_file"] = session_file
    st.session_state["username"] = username
    if load_existing_session and os.path.exists(session_file):
        data = load_session(session_file)
        # if "username" in data:
        #     del data["username"]
        update_session_profile(data)
    if diff_session_file is not None:
        data = json.load(diff_session_file)
        # if "username" in data:
        #     del data["username"]
        update_session_profile(data)
        save_session(session_file, st.session_state)

# @st.cache(suppress_st_warning=True, allow_output_mutation=True)
def step2_setup_image_label_path():
    st.subheader("1. Set Image Path")
    image_path_col1, image_path_col2, image_path_col3 = st.columns([2, 0.5, 2])

    image_folder_path = image_path_col1.text_input(
        "Please input the path to the folder containing images",
        key="image_folder",
        on_change=update_session_file,
    )
    image_path_col2.text("OR")
    image_csv_file = image_path_col3.text_input(
        "Please input the path to csv file with 'img_path' column to the images",
        key="image_csv_file",
        on_change=update_session_file,
    )

    st.subheader("2. Set the Label Path")
    label_path = st.text_input(
        "Please input the path to save the label (json format)",
        key="label_path",
        on_change=update_session_file,
    )

    img_df = pd.DataFrame(columns=["img_path", "img_name"])
    label_df = pd.DataFrame(columns=["label"])
    label_dict = {}
    derived_labels = ""

    if image_folder_path != "":
        if os.path.exists(image_folder_path) is False:
            st.error("%s doesn't exist" % (image_folder_path))
        img_paths = [
            os.path.join(image_folder_path, _)
            for _ in os.listdir(image_folder_path)
            if "csv" not in _
        ]
        img_names = [_.split("/")[-1].split(".")[0] for _ in img_paths]
        img_df = pd.DataFrame.from_dict(
            {
                "img_path": img_paths,
                "img_name": img_names,
            }
        )
    elif image_csv_file != "":
        if os.path.exists(image_csv_file) is False:
            st.error(image_csv_file + " doesn't exist")
        else:
            img_df = get_image_csv(image_csv_file)
    else:
        st.error("image path is not specified")

    if label_path == "":
        st.error("label_path is not specified, please set the path to save the labels")
    else:
        if os.path.exists(label_path):
            label_dict = load_label_json(label_path)
            label_df = pd.Series(label_dict).to_frame(name="label")
            json_labels = []
            for _, val in label_dict.items():
                json_labels.extend(val.split(","))
            derived_labels = ",".join(sorted(np.unique(json_labels)))

    return img_df, label_df, label_dict, label_path, derived_labels

def step3_display_and_label(derived_labels, state, img_df, label_dict, label_path):
    st.subheader("3. Label the images one by one")
    (
        img_width_col,
        img_height_col,
        label_input_buff,
        save_multilabels_to_session_col,
    ) = st.columns([1, 1, 3, 1])
    target_width = img_width_col.number_input(
        "Image width", min_value=10, max_value=1000, value=300
    )
    target_height = img_height_col.number_input(
        "height", min_value=10, max_value=1000, value=300
    )
    union_labels = []
    if "multiple_labels" in st.session_state:
        union_labels = ",".join(
            sorted(
                list(
                    set(derived_labels).union(
                        st.session_state["multiple_labels"].split(",")
                    )
                )
            )
        )
        multiple_labels = label_input_buff.text_input(
            "Multi-Label Options (seperate by ,)",
            value=st.session_state["multiple_labels"],
        )
    else:
        if len(derived_labels) == 0:
            multiple_labels = label_input_buff.text_input(
                "Multi-Label Options (seperate by ,)", value="dog,cat"
            )
        else:
            multiple_labels = label_input_buff.text_input(
                "Multi-Label Options (seperate by ,)", value=derived_labels
            )
    save_multilabels_to_session_col.button(
        "Save Multi-label Options to Session File",
        on_click=update_multiple_labels,
        args=(multiple_labels,),
    )

    target_labels = [_.strip() for _ in multiple_labels.split(",")]

    col1, col2, col3 = st.columns([1.2, 3, 2])
    with col1:

        if "current_row" not in state or state["current_row"] is None:
            state["current_row"] = 0
        if "current_row" in state and state["current_row"] is not None:
            if state["current_row"] < 0:
                state["current_row"] = len(img_df) - 1
            elif state["current_row"] >= len(img_df):
                state["current_row"] = 0

        prev_click = st.button("previous")
        next_click = st.button("next")
        current_row = state["current_row"]
        if prev_click is True:
            current_row -= 1
            if current_row < 0:
                current_row = len(img_df) - 1
        if next_click is True:
            current_row += 1
            if current_row >= len(img_df):
                current_row = 0
        current_row = st.selectbox(
            "Select an image for labeling", img_df.index, current_row
        )
        if current_row is not None:
            state["current_row"] = current_row

    with col2:
        img_path = img_df.loc[current_row, "img_path"]
        image = Image.open(img_path)
        image = image.resize((target_width, target_height))
        image_name = img_path.split("/")[-1].split(".")[0]
        st.image(image, caption=image_name)

    current_labels = img_df.loc[current_row, "label"].split(",")
    selected_labels = col3.multiselect(
        "Current Label", options=target_labels + [""], default=current_labels
    )
    label_dict[img_path] = ",".join(sorted([_ for _ in selected_labels if _ != ""]))
    col3.button(
        "Update Label",
        on_click=update_label_json,
        args=(label_path, label_dict),
    )
    pass

def step4_label_stats(img_df):
    #### Step 3
    st.subheader("4. Label Stats")
    fig, axes = plt.subplots(2, 2)
    fig.set_figheight(8)
    fig.set_figwidth(12)
    img_df["label_category"] = img_df["label"].apply(lambda x: label_category(x))
    img_df.groupby("label_category").count()["img_path"].plot(
        kind="barh", ax=axes[0][0]
    )
    axes[0][0].set_title("Label Category")

    all_labels = extract_all_labels(img_df["label"].values)
    w = Counter(all_labels)
    axes[0][1].barh(list(w.keys()), list(w.values()))
    axes[0][1].set_title("All Labels")

    single_labels = extract_all_labels(
        img_df[img_df["label_category"] == "Single Label"]["label"].values
    )
    w2 = Counter(single_labels)
    axes[1][0].barh(list(w2.keys()), list(w2.values()))
    axes[1][0].set_title("Single Labels")

    multi_labels = extract_multilabels(
        img_df[img_df["label_category"] == "Multi Label"]["label"].values
    )
    w3 = Counter(multi_labels)
    axes[1][1].barh(list(w3.keys()), list(w3.values()))
    axes[1][1].set_title("Multi Labels")

    st.pyplot(fig)

def main():

    ### Instruction
    st.header("Multi-label Image Classification Labeling Tool")
    st.markdown(
        """
        This is an image classification labeling APP created with Streamlit. It supports multi-labels for the same image.
        Please follow the instructions to login, session setup, load, label images and save your labels

    """
    )

    if check_password():

        # st.write(st.session_state)

        ## User Session Handle
        session_ready = False
        if "session_file" in st.session_state and os.path.exists(
            st.session_state["session_file"]
        ):
            session_ready = True
            data = load_session(st.session_state["session_file"])
            # if "username" in data:
            #     del data["username"]
            update_session_profile(data)
            st.write(
                "Current session are loaded from %s"
                % (st.session_state["session_file"])
            )
        else:
            st.subheader("Set Up Session")
            step1_setup_session()

        if session_ready:
            ## Step 1
            (
                img_df,
                label_df,
                label_dict,
                label_path,
                derived_labels,
            ) = step2_setup_image_label_path()

            img_df = dataframe_join_image_label(img_df, label_df)

            ## Display and filter loaded table
            state = get_state()
            st.text("Set a filter for the images (Optional)")
            filter1_col1, filter1_col2 = st.columns([1, 4])
            filter1_col = filter1_col1.text_input(
                "Filter Column Name", key="filter1_col"
            )
            if filter1_col in img_df and is_numeric_dtype(img_df[filter1_col]):
                min_val, max_val = float(min(0, img_df[filter1_col].min())), float(
                    max(1, img_df[filter1_col].max())
                )
                if "filter1_values" in state:
                    filter1_values = state["filter1_values"]
                else:
                    filter1_values = (min_val, max_val)

                val1, val2 = filter1_col2.slider(
                    "filter 1 range",
                    min_val,
                    max_val,
                    filter1_values,
                    key="filter1_values",
                )
                img_df = img_df[img_df[filter1_col].apply(lambda x: val1 <= x <= val2)]

            st.dataframe(img_df, width=800, height=200)

            if len(img_df) == 0:
                st.error("No images are selected for labels")
            else:
                #### Step two
                step3_display_and_label(
                    derived_labels, state, img_df, label_dict, label_path
                )
                step4_label_stats(deepcopy(img_df))

if __name__ == "__main__":
    main()


##########################################
#Constants

c = 299792458
##########################################

'''Just some formulas of relativistic mechanics'''
                                            #YuPie

class RelativisticEnergy():

    def energyVelocity(V, m_0):
        '''Velocity must be lower then speed of light'''
        if V >= c:
            print("Incorrect velocity")
        else:
            m = m_0/(math.sqrt(1-((V**2)/(c**2))))
            E = m*(c**2)
            print("Relativistic of your object is " + str(E) + " J")

    def energyCalm(m):
        '''Just enter mass of your object'''
        E = m*(c**2)
        print("Relativistic of your object is " + str(E) + " J")


#encoding:utf-8
import tensorflow as tf
from Parameters import Parameters as pm
from data_processing import *

class TextCnn(object):

    def __init__(self):

        self.input_x = tf.placeholder(tf.int32, shape=[None, pm.seq_length], name='input_x')
        self.input_y = tf.placeholder(tf.float32, shape=[None, pm.num_classes], name='input_y')
        self.keep_pro = tf.placeholder(tf.float32, name='drop_out')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')
        self.cnn()

    def cnn(self):
        with tf.device('/cpu:0'), tf.name_scope('embedding'):
            self.embedding = tf.get_variable("embeddings", shape=[pm.vocab_size, pm.embedding_size],
                                             initializer=tf.constant_initializer(pm.pre_trianing))
            embedding_input = tf.nn.embedding_lookup(self.embedding, self.input_x)
            self.embedding_expand = tf.expand_dims(embedding_input, -1)

        pooled_outputs = []
        for i, filter_size in enumerate(pm.kernel_size):
            with tf.name_scope("conv-maxpool-%s" % filter_size):
                filter_shape = [filter_size, pm.embedding_size, 1, pm.num_filters]
                w = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name= 'w')
                b = tf.Variable(tf.constant(0.1, shape=[pm.num_filters]), name='b')
                conv = tf.nn.conv2d(self.embedding_expand, w, strides=[1, 1, 1, 1], padding='VALID', name='conv')
                h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')

                pooled = tf.nn.max_pool(h, ksize=[1, pm.seq_length - filter_size + 1, 1, 1],
                                        strides=[1, 1, 1, 1], padding='VALID', name='pool')
                pooled_outputs.append(pooled)

        num_filter_total = pm.num_filters * len(pm.kernel_size)
        self.h_pool = tf.concat(pooled_outputs, 3)
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filter_total])

        with tf.name_scope('dropout'):
            self.h_drop = tf.nn.dropout(self.h_pool_flat, self.keep_pro)

        with tf.name_scope('output'):
            w = tf.get_variable("w", shape=[num_filter_total, pm.num_classes],
                                initializer=tf.contrib.layers.xavier_initializer())

            b = tf.Variable(tf.constant(0.1, shape=[pm.num_classes]), name='b')

            self.scores = tf.nn.xw_plus_b(self.h_drop, w, b, name='scores')
            self.pro = tf.nn.softmax(self.scores)
            self.predicitions = tf.argmax(self.pro, 1, name='predictions')

        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)
            self.loss = tf.reduce_mean(losses)  #对交叉熵取均值非常有必要

        with tf.name_scope('optimizer'):
            #退化学习率 learning_rate = lr*(0.9**(global_step/10);staircase=True表示每decay_steps更新梯度
            #learning_rate = tf.train.exponential_decay(self.config.lr, global_step=self.global_step,
                                                       #decay_steps=10, decay_rate=self.config.lr_decay, staircase=True)
            #optimizer = tf.train.AdadeltaOptimizer(learning_rate)
            #self.optimizer = optimizer.minimize(self.loss, global_step=self.global_step) #global_step 自动+1
            #no.2
            optimizer = tf.train.AdamOptimizer(pm.lr)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))#计算变量梯度，得到梯度值,变量
            gradients, _ = tf.clip_by_global_norm(gradients, pm.clip)
            #对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            #global_step 自动+1

        with tf.name_scope('accuracy'):
            correct_predictions = tf.equal(self.predicitions, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, 'float32'), name='accuracy')

    def feed_data(self, x_batch, y_batch, keep_prob):
        feed_dict = {
            self.input_x: x_batch,
            self.input_y: y_batch,
            self.keep_pro: keep_prob
        }
        return feed_dict

    def evaluate(self, sess, x, y):
        batch_eva = batch_iter(x, y, pm.batch_size)
        for x_batch, y_batch in batch_eva:
            feed_dict = self.feed_data(x_batch, y_batch, 1.0)
            loss, accuracy = sess.run([self.loss, self.accuracy], feed_dict=feed_dict)
        return loss, accuracy


"""Count the number of elements in a list
Input: nums = [2,2,1,1,1,2,2,3]
o/p:[1,3,4]
"""
nums = [2,2,1,1,1,2,2,3]
new=[]
for i in nums:
    new.append(nums.count(i))
new[:]=set(new)
new


import cv2
import sys
import numpy as np
from sklearn.model_selection import train_test_split
import pandas as pd
import os
import time
from sklearn.ensemble import RandomForestClassifier
import pickle
from Classifier import Classifier

def getFilePath (folder):
    video_files = os.listdir(folder)
    return [a + '/' + b for a, b in zip([folder] * len(video_files), video_files)]

def main(new_model_training = False):
    file_paths = getFilePath('input_data')
    file_labels = [i.split('_')[2] for i in file_paths]
    x_train, x_unused, y_train, y_unused = \
        train_test_split(file_paths, file_labels,
                         test_size=0.5, random_state=0, stratify = file_labels)

    cls = Classifier()

    if new_model_training:
        cls.add_training_data(x_train, y_train)
        rf = RandomForestClassifier(n_estimators=300)  # can be replaced with any other ML models
        cls.train(learner=rf)
        """
        with open ('model.pkl', 'wb') as file:
            pickle.dump(rf, file)
        with open ('scaler.pkl', 'wb') as file:
            pickle.dump(cls.scaler, file)
        """

    else:
        with open ('model.pkl', 'rb') as file:
            rf = pickle.load(file)
        with open ('scaler.pkl', 'rb') as file:
            scaler = pickle.load(file)
        cls.scaler = scaler

    predictions = cls.predict(learner=rf, x_test=x_unused)

    accuracy = ([i[2] for i in predictions.index.str.split('_')] == predictions).sum() / len(predictions)
    print ('accuracy:', accuracy)

    # extract test results
    """
    test_predictions = pd.DataFrame([i[2] for i in pd.Series(predictions.index).str.split('_')], columns=['actual'])
    test_predictions['prediction'] = predictions.reset_index()['predictions']
    test_predictions.groupby(['actual', 'prediction']).size()
    """

if __name__ == "__main__":
    if sys.argv[1:][0] == '--train=True':
        print ('Training = True activated')
        main(new_model_training=True)
    else:
        print('Training = False activated')
        main()


import pandas as pd
import json
import datetime

print(datetime.datetime.now())

with open("data/json/reformatted_nonli.json", "r") as f:
    data_reformatted_nonli = json.load(f)

def json_to_pandas(data_json, schema, column):

    df = pd.DataFrame(columns = schema)

    for i, item in enumerate(data_json):

        print('{} / {}'.format(i+1, len(data_json)), end="\r")

        df_batch = pd.DataFrame(columns = schema)

        for item_column in item[column]:
            row = [item['org_uuid'], *[item_column[title] for title in schema[1:]]]
            df_batch = pd.concat(
                [
                    df_batch,
                    pd.DataFrame([row], columns=schema)
                ]
            )

        df = pd.concat([df, df_batch])

    print('DataFrame {0} size : {1}'.format(column, df.shape[0]))

    return df

education_schema = [
        'org_uuid',
        'education_school_name',
        'education_field_of_study',
        'education_description',
        'education_degree',
        'education_start_date_year',
        'education_end_date',
        'education_school_link'
    ]

experience_schema = [
        'org_uuid',
        'experience_title',
        'experience_start_date',
        'experience_end_date',
        'experience_headcount_range',
        'experience_company_name',
        'experience_company_link',
        'experience_company_id_link',
        'experience_industry',
        'experience_location',
        'experience_description'
    ]

skills_schema = [
        'org_uuid',
        'skill',
    ]

json_to_pandas(
    data_json=data_reformatted_nonli,
    schema=education_schema,
    column='educations'
).to_csv('data/uuid_x_education_fields.csv')

json_to_pandas(
    data_json=data_reformatted_nonli,
    schema=experience_schema,
    column='experiences'
).to_csv('data/uuid_x_experience_fields.csv')

json_to_pandas(
    data_json=data_reformatted_nonli,
    schema=skills_schema,
    column='skills'
).to_csv('data/uuid_x_skills_fields.csv')


from django.contrib import admin

# Register your models here.
from campaign.models import Campaign, CampaignPackage, CampaignSubscriber

@admin.register(Campaign)
class CampaignAdmin(admin.ModelAdmin):
    pass

@admin.register(CampaignPackage)
class CampaignPackageAdmin(admin.ModelAdmin):
    list_display = [
        'campaign',
        'name',
        'price',
        'starts',
        'ends',
        'discount_percentage_off',
        'discount_amount_off',
        'discount_starts',
        'discount_expires',
    ]

@admin.register(CampaignSubscriber)
class CampaignSubscriberAdmin(admin.ModelAdmin):
    list_display = [
        'name',
        'guid'
    ]


import tensorflow as tf
import os
import numpy as np
import cv2
import shutil

def move_file():
    print()

class classify():
    data = []
    tmp = os.path.join("D:/Study/2020-Crawler-Dataton/main/Selenium/Instagram/tmp")
    filenames = []
    def __init__(self):
        print()

    def predict(self, data,filename):
        data = data/255.
        data = tf.image.resize([data], (224, 224))
        if os.path.isfile("./model.h5"):
            base_model = tf.keras.applications.ResNet50(input_shape=(224, 224, 3))
            base_model = tf.keras.models.Model(base_model.inputs, base_model.layers[-2].output)
            x = base_model.output
            pred = tf.keras.layers.Dense(3, activation='softmax')(x)

            model = tf.keras.models.Model(inputs=base_model.input, outputs=pred)
            opt = tf.keras.optimizers.Adam(learning_rate=0.0001)

            model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['acc'])
            model = tf.keras.models.load_model("./model.h5")
            predict = np.argmax(model.predict(data))
            dir = "./"+str(predict)+"/"+filename
            shutil.move(self.tmp+"/"+filename, dir+filename)

    def load_img(self,folder):
        data_path = os.listdir(folder)
        for i in range(len(data_path)):
            img = cv2.imread(folder + "/" + data_path[i])
            img = cv2.resize(img, (256, 256))
            self.data.append(img)
            self.filenames.append(data_path[i])

        return True

    def run(self):
        if self.load_img(self.tmp):
            self.data = np.array(self.data)
            for i in range(np.size(self.data, axis=0)):
                self.predict(self.data[i],self.filenames[i])

a = classify()
a.run()


import json
import os.path
from utils.eval_metrics import moses_multi_bleu
import glob as glob
import numpy as np
import jsonlines
from tabulate import tabulate
from tqdm import tqdm

def compute_prf_SMD(gold, pred, global_entity_list):#, kb_plain=None):
    # local_kb_word = [k[0] for k in kb_plain]
    TP, FP, FN = 0, 0, 0
    if len(gold)!= 0:
        count = 1
        for g in gold:
            if g in pred:
                TP += 1
            else:
                FN += 1
        for p in set(pred):
            if p in global_entity_list:# or p in local_kb_word:
                if p not in gold:
                    FP += 1
                    print(p)
        # print("TP",TP)
        # print("FP",FP)
        # print("FN",FN)
        precision = TP / float(TP+FP) if (TP+FP)!=0 else 0
        # print(precision)
        recall = TP / float(TP+FN) if (TP+FN)!=0 else 0
        # print(recall)
        F1 = 2 * precision * recall / float(precision + recall) if (precision+recall)!=0 else 0
        # print(F1)
    else:
        precision, recall, F1, count = 0, 0, 0, 0
    return F1, count

def get_global_entity_KVR():
    with open('data/SMD/kvret_entities.json') as f:
        global_entity = json.load(f)
        global_entity_list = []
        for key in global_entity.keys():
            if key != 'poi':
                global_entity_list += [item.lower().replace(' ', '_') for item in global_entity[key]]
            else:
                for item in global_entity['poi']:
                    global_entity_list += [item[k].lower().replace(' ', '_') for k in item.keys()]
        global_entity_list = list(set(global_entity_list))
    return global_entity_list

def score_SMD(model,file_to_score):
    genr_json = json.load(open(file_to_score))
    global_entity_list = get_global_entity_KVR()
    GOLD, GENR = [], []
    F1_score = []
    F1_domain = {"navigate":[],"weather":[],"schedule":[]}
    num_lines = sum(1 for line in open("data/SMD/test.txt",'r'))
    with open("data/SMD/test.txt",'r') as f:
        idd = 0
        for line in tqdm(f,total=num_lines):
            line = line.strip()
            if line:
                if '#' in line:
                    idd += 1
                    i = 0
                    line = line.replace("#","")
                    task_type = line
                    continue

                _, line = line.split(' ', 1)
                if '\t' in line:
                    u, r, gold_ent = line.split('\t')
                    # print(gold_ent, genr_json[str(idd)][i]['text'].split(" "))
                    F1, count = compute_prf_SMD(eval(gold_ent), genr_json[str(idd)][i]['text'].replace("."," .").replace("'"," '").replace("?"," ?").replace(","," ,").replace("!"," !").replace("  "," ").split(" "), global_entity_list)
                    if(count==1):
                        F1_score.append(F1)
                        F1_domain[task_type].append(F1)
                    generated = genr_json[str(idd)][i]['text']
                    # print("USER",u)
                    # print("GOLD",r)
                    # print("GENE",generated)
                    GOLD.append(r)
                    GENR.append(generated.replace(",",""))
                    # print("ENTI",eval(gold_ent))
                    # print(F1)
                    # print()
                    # print()
                    # print()
                    i += 1
                    # input()

    BLEU = moses_multi_bleu(np.array(GENR),np.array(GOLD))

    return {"Model":model,
            "BLEU":BLEU,
            "F1":100*np.mean(F1_score),
            "F1 navigate":100*np.mean(F1_domain["navigate"]),
            "F1 weather":100*np.mean(F1_domain["weather"]),
            "F1 schedule":100*np.mean(F1_domain["schedule"])}

rows = [
        {"Model":"KVRet","BLEU":13.2,"F1":48.0,"F1 navigate":44.5,"F1 weather":53.3,"F1 schedule":62.9},
        {"Model":"MLMN","BLEU":17.1,"F1":55.1,"F1 navigate":41.3,"F1 weather":47.0,"F1 schedule":68.3},
        # {"Model":"---","BLEU":"---","F1":"---","F1 navigate":"---","F1 weather":"---","F1 schedule":"---"},
        {"Model":"Mem2Seq","BLEU":12.2,"F1":33.40,"F1 navigate":20.00,"F1 weather":49.30,"F1 schedule":32.80},
        {"Model":"KBRet","BLEU":13.9,"F1":53.7,"F1 navigate":54.5,"F1 weather":52.2,"F1 schedule":55.6},
        {"Model":"GLMP","BLEU":13.9,"F1":60.7,"F1 navigate":54.6,"F1 weather":56.5,"F1 schedule":72.5},
        {"Model":"DFF","BLEU":14.4,"F1":62.7,"F1 navigate":57.9,"F1 weather":57.6,"F1 schedule":73.1},
        ]

# for f in glob.glob("runs/*"):
#     if("SMD" in f and os.path.isfile(f+'/result.json')):
#         d_name,model,_,graph,_,adj,_,edge,_,unilm,_,flattenKB,_,hist_L,_,*_ = f.split("/")[1].split("_")
#         st = model
#         if(eval(graph)): st+= "+NODE "
#         if(eval(adj)): st+= "+ADJ "
#         if(eval(edge)): st+= "+EDGES "
#         if(eval(unilm)): st+= "+UNI "
#         if(eval(flattenKB)): st+= "+KB "
#         rows.append(score_SMD(st,f+'/result.json'))

# baseline
rows.append({"Model":"gpt2","BLEU":14.72,"F1":39.11,"F1 navigate":23.41,"F1 weather":53.74,"F1 schedule":52.26})

# previous results
rows.append({"Model":"gpt2+KB","BLEU":14.72,"F1":57.50,"F1 navigate":49.00,"F1 weather":58.63,"F1 schedule":71.13})
rows.append({"Model":"T10","BLEU":11.24,"F1":52.88,"F1 navigate":50.26,"F1 weather":51.64,"F1 schedule":58.62})
rows.append({"Model":"T25","BLEU":12.26,"F1":55.00,"F1 navigate":50.46,"F1 weather":52.91,"F1 schedule":64.87})
rows.append({"Model":"T50","BLEU":13.01,"F1":56.43,"F1 navigate":50.04,"F1 weather":54.25,"F1 schedule":69.60})
rows.append({"Model":"T75","BLEU":13.67,"F1":58.79,"F1 navigate":52.56,"F1 weather":56.39,"F1 schedule":71.89})
rows.append({"Model":"T100","BLEU":0,"F1":0,"F1 navigate":53.53,"F1 weather":0,"F1 schedule":72.58})

rows.append(score_SMD("GPT2+FINE",'runs/SMD_gpt2_graph_False_adj_False_edge_False_unilm_False_flattenKB_False_historyL_1000000000_lr_6.25e-05_epoch_10_weighttie_False_kbpercentage_0_layer_12/all_results.json'))

print(tabulate(rows,headers="keys",tablefmt='latex',floatfmt=".2f"))


import pandas as pd
from data.data import get_data_parquet,get_data_xslx
from sklearn.preprocessing import MinMaxScaler
import numpy as np
from conf.conf import logging

def preprocess_movies_rd(movies_rd: pd.DataFrame) -> pd.DataFrame:
    """
    Makes preprocessing of parsed dataframe with ranking and description data for movies.
    Returns usable df for further caclulations
    """

    logging.info('Preprcessing DataFrame')

    movies_rd['rating'] = [str(x) for x in movies_rd['rating'] .values]
    movies_rd['rating'] = [x.replace('dict_values([','') for x in movies_rd['rating'].values]
    movies_rd['rating'] = [x.replace('])','') for x in movies_rd['rating'].values]
    movies_rd['rating'] = [float(x) for x in movies_rd['rating'] .values]
    movies_rd['description'] = [str(x) for x in movies_rd['description'] .values]

    logging.info('DataFrame preprocessed')

    return movies_rd

def get_int_coef(interactions: pd.DataFrame) -> pd.DataFrame:
    """
    Calculates the coefficient of interactions as number of interactions with the film
    """

    logging.info('Getting Interactions Coeficient')

    interactions['count'] = interactions['count']= [1 for x in interactions['user_id']]
    int_coef = interactions.groupby(['user_id']).sum()
    int_coef['interactions_coefficient'] = np.clip(int_coef['count'],1,20)
    int_coef['interactions_coefficient'] = np.log(int_coef['interactions_coefficient'])
    int_coef = int_coef.drop('count',axis=1)
    int_coef = int_coef[['interactions_coefficient','day']]
    int_coef = int_coef.drop('day',axis=1)
    scaler = MinMaxScaler()
    int_coef['interactions_coefficient'] = scaler.fit_transform(np.array(int_coef['interactions_coefficient']).reshape(-1,1))

    logging.info('Interactions Coeficient is ready')

    return int_coef

def get_popularity(interactions:pd.DataFrame) -> pd.DataFrame:
    """
    Calculates the popularity as number of users who watched the film
    """

    logging.info('Counting Popularity')

    interactions['count']= [1 for x in interactions['movie_id']]
    popularity = interactions.groupby(['movie_id']).sum()
    popularity['num_users_watched'] = popularity['count']
    popularity = popularity.drop('count',axis=1)
    popularity = popularity[['watch_duration_minutes','num_users_watched']]

    logging.info('Popularity is ready')

    return popularity

def get_avg(interactions:pd.DataFrame) -> pd.DataFrame:
    """
    Calculates the average watch coefficient as time watched divided by length of the film
    """

    logging.info('Counting Average Time Watched')

    avg = interactions.groupby(['movie_id']).mean()
    avg = avg[['watch_duration_minutes','day']]
    avg['avg_watch_coef'] = avg['watch_duration_minutes']
    avg = avg.drop(['watch_duration_minutes','day'],axis=1)

    logging.info('Average Time Watched is ready')

    return avg

def get_movies_upd(movies_md:pd.DataFrame, popularity:pd.DataFrame, avg:pd.DataFrame,movies_rd:pd.DataFrame) -> pd.DataFrame:
    """
    Merges the movies dataframe with the popularity and average watch coefficient dataframes
    """

    logging.info('Merging Movies Md, Populatiry, Average Time Watched, Rating, Description')

    movies_md = movies_md.merge(popularity, how='right', on= 'movie_id')
    movies_md['time_watched'] = np.clip(movies_md['watch_duration_minutes']/movies_md['duration'],0,1)
    movies_md = movies_md.merge(avg, how='right', on= 'movie_id')
    movies_md = movies_md.merge(movies_rd, how='left',on='title')

    logging.info('Merge Succesful')

    return movies_md

def convert_columns(df:pd.DataFrame, columns_list:list) -> pd.DataFrame:
    """
    Deleted unnecessary characters from text
    """

    logging.info('Clearing up the columns')

    for column in columns_list:
        df[column] = [str(x) for x in df[column].values]
        df[column] = [x.replace('"','') for x in df[column].values]
        df[column] = [x.replace('[','') for x in df[column].values]
        df[column] = [x.replace(']','') for x in df[column].values]
        df[column] = [x.replace(',',' ') for x in df[column].values]

    logging.info('Columns cleared up')

    return df

def get_desc(movies_upd:pd.DataFrame) -> pd.DataFrame:
    """
    Finalizes the description of the film to single text containing all the necessary information
    """

    logging.info('Creating the description')

    movies_upd['desc_md'] = movies_upd['title'] + ' ' + movies_upd['entity_type'] + ' ' + movies_upd['genres'] + ' ' + movies_upd['director']
    movies_upd = movies_upd.dropna(subset=['desc_md'])

    logging.info('Descrition is ready')

    return movies_upd

def get_score(df: pd.DataFrame) -> pd.DataFrame:
    """
    Returns final score of the movie
    """

    logging.info('Calculating score for movies')

    df['rating'] = df['rating'].replace(-1,0)
    df['time_watched'] = df['time_watched'].replace(np.nan,0)
    scaler = MinMaxScaler()
    df['time_watched'] = scaler.fit_transform(np.array(df['time_watched']).reshape(-1,1))
    df['num_users_watched'] = scaler.fit_transform(np.array(df['num_users_watched']).reshape(-1,1))
    df['avg_watch_coef'] = scaler.fit_transform(np.array(df['avg_watch_coef']).reshape(-1,1))
    df['rating'] = scaler.fit_transform(np.array(df['rating']).reshape(-1,1))
    df['score'] = df['time_watched']*0.1 + df['num_users_watched']*0.4+df['avg_watch_coef']*0.1 + df['rating']*0.4
    #df['score'] = np.clip(df['score'],0,100)
    df['score'] = scaler.fit_transform(np.array(df['score']).reshape(-1,1))

    logging.info('Score is ready')

    return df

def get_dur(movies_md: pd.DataFrame,interactions: pd.DataFrame)-> pd.DataFrame:
    """
    Calculates how long a certain user have watched the film
    """

    logging.info('Calculating how long a user wacthed a certain film')

    interactions = interactions.merge(movies_md,how='left',on='movie_id')
    interactions['watched'] = interactions['watch_duration_minutes_x']/interactions['duration']
    interactions['watched'] = np.clip(interactions['watched'],0,1)
    interactions = interactions.replace(np.nan,interactions['watched'].mean())
    interactions =  interactions[['year','month','day','user_id','movie_id','watch_duration_minutes_x','watched']]
    interactions.rename({'watch_duration_minutes_x':'watch_duration_minutes'})

    logging.info('Calculation is done')

    return interactions


top_heights = [6, 9, 5, 7, 4]

def get_receiver_top_orders(heights):
    nth_top = [0] * len(heights)
    while heights:
        height = heights.pop()
        #heights = [6,9,5,7]
        for idx in range(len(heights)-1, 0,-1):
            if heights[idx]>height:
                nth_top[len(heights)] =idx+1
                break

    return nth_top

print(get_receiver_top_orders(top_heights))
# [0, 0, 2, 2, 4] 가 반환되어야 한다!


#!/usr/bin/python
from pwn import *
key1 = 0x00008cdc
key1 += 8 # 0x00008ce4

key2 = 0x00008d04
key2 += 4
key2 += 4

key3 = 0x00008d80

key = key1 + key2 + key3
print(f"Key 1 : {hex(key1)}")
print(f"Key 2 : {hex(key2)}")
print(f"Key 3 : {hex(key3)}")
print(key)

#I am getting an error while ssh-ing into this particular machine so no connection using pwntools