from datetime import date
from datetime import datetime
import time
import logging
import os
from util import calc_open_mac
from util import calc_unsubscribe_mac
from util import calc_unsubscribe_mac_old
from util import calc_url_mac
import urllib.request, urllib.parse, urllib.error
from urllib.parse import urlparse
import json
import subprocess
import sys
import csv
from html.parser import HTMLParser

from django.conf import settings
from django.contrib import messages
from django.core.exceptions import PermissionDenied
from django.core.paginator import EmptyPage
from django.core.paginator import PageNotAnInteger
from django.core.paginator import Paginator
from django.urls import reverse
from django.db.models import Max, Min, Q
from django.http import HttpResponse
from django.http import HttpResponseForbidden
from django.http import HttpResponseRedirect
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.views.generic.base import TemplateView
from django.views.generic.base import View
from django.views.generic.edit import CreateView
from django.views.generic.edit import DeleteView
from django.views.generic.edit import FormView
from django.views.generic.edit import UpdateView
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.forms.utils import ErrorList
from django.contrib.staticfiles.storage import staticfiles_storage

from manager.forms import CampaignForm
from manager.forms import EmailSearchForm
from manager.forms import EmailCreateUpdateForm
from manager.forms import EmailInstantSendForm
from manager.forms import ExportCleanupForm
from manager.forms import PreviewInstanceLockForm
from manager.forms import RecipientAttributeCreateForm
from manager.forms import RecipientAttributeUpdateForm
from manager.forms import RecipientAttributeFormSet
from manager.forms import RecipientCreateUpdateForm
from manager.forms import RecipientCSVImportForm
from manager.forms import RecipientGroupSearchForm
from manager.forms import RecipientGroupCreateForm
from manager.forms import RecipientGroupUpdateForm
from manager.forms import RecipientSearchForm
from manager.forms import RecipientSubscriptionsForm
from manager.forms import SegmentForm
from manager.forms import IncludeSegmentRuleFormset
from manager.forms import ExcludeSegmentRuleFormset
from manager.forms import SettingCreateUpdateForm
from manager.forms import SubscriptionCategoryForm
from manager.models import Campaign
from manager.models import Email
from manager.models import Instance
from manager.models import InstanceOpen
from manager.models import PreviewInstance
from manager.models import RecipientAttribute
from manager.models import Recipient
from manager.models import RecipientGroup
from manager.models import Segment
from manager.models import SegmentRule
from manager.models import Setting
from manager.models import StaleRecord
from manager.models import SubprocessStatus
from manager.models import SubscriptionCategory
from manager.models import URL
from manager.models import URLClick
from manager.utilities.email_sender import EmailSender
from manager.utilities.s3_helper import AmazonS3Helper

log = logging.getLogger(__name__)

##
# Mixins
##
class SortSearchMixin(object):
    def get_queryset(self):
        queryset = super(SortSearchMixin, self).get_queryset()

        # sort parameter
        self._sort = None
        self._order = None
        self._order_change = 'asc'
        self._search_query = ''

        if self.request.GET.get('sort'):
            self._sort = self.request.GET.get('sort')

        if self.request.GET.get('order'):
            self._order = self.request.GET.get('order')

        if self.request.GET.get('search_query'):
            self._search_query = self.request.GET.get('search_query')

        # search parameter (search_form defined in View)
        self._search_valid = self.search_form.is_valid()

        if self._search_valid:
            kwargs = {
                '{0}__icontains'.format(self.search_field): self.search_form.cleaned_data['search_query']
            }
            queryset = queryset.filter(**kwargs)

        if self._sort and self._order == 'des':
            self._order_change = 'asc'
            return queryset.order_by('-{0}'.format(self._sort))
        elif self._sort and self._order == 'asc':
            self._order_change = 'des'
            return queryset.order_by(self._sort)
        else:
            return queryset

    def generate_query_params(self, page, previous = True):
        query_mappings = {}

        if previous and page and page.has_previous():
            query_mappings['page'] = page.previous_page_number()

        if not previous and page and page.has_next():
            query_mappings['page'] = page.next_page_number()

        if self._search_query and self._search_query != '':
            query_mappings['search_query'] = self._search_query

        if self._sort and self._sort != '':
            query_mappings['sort'] = self._sort

        if self._order and self._order != '':
            query_mappings['order'] = self._order

        return '?{0}'.format(urllib.parse.urlencode(query_mappings))

    def get_context_data(self, **kwargs):
        context = super(SortSearchMixin, self).get_context_data(**kwargs)
        context['sort'] = self._sort
        context['order_change'] = self._order_change
        context['search_query'] = self._search_query

        query_mapping = {}

        if 'page_obj' in context:
            page = context['page_obj']

            if page and page.has_previous():
                context['previous_url'] = self.generate_query_params(page, True)

            if page and page.has_next():
                context['next_url'] = self.generate_query_params(page, False)

        return context

class EmailsMixin(object):
    def get_context_data(self, **kwargs):
        context = super(EmailsMixin, self).get_context_data(**kwargs)
        context['section'] = 'emails'
        return context

class RecipientGroupsMixin(object):
    def get_context_data(self, **kwargs):
        context = super(RecipientGroupsMixin, self).get_context_data(**kwargs)
        context['section'] = 'recipientgroups'
        return context

class RecipientsMixin(object):
    def get_context_data(self, **kwargs):
        context = super(RecipientsMixin, self).get_context_data(**kwargs)
        context['section'] = 'recipients'
        return context

class SettingsMixin(object):
    def get_context_data(self, **kwargs):
        context = super(SettingsMixin, self).get_context_data(**kwargs)
        context['section'] = 'settings'
        return context

class SubscriptionsMixin(object):
    def get_context_data(self, **kwargs):
        context = super(SubscriptionsMixin, self).get_context_data(**kwargs)
        context['section'] = 'subscriptions'
        return context

class OverviewListView(ListView):
    template_name = 'manager/instructions.html'
    context_object_name = 'emails'

    def get_queryset(self):
        return Email.objects.sending_today().order_by('send_time')

    def get_context_data(self, **kwargs):
        context = super(OverviewListView, self).get_context_data(**kwargs)

        try:
            contact_info = Setting.objects.get(
                name='office_hours_contact_info')
            context['office_hours_contact_info'] = contact_info.value
        except Setting.DoesNotExist:
            pass

        try:
            contact_info = Setting.objects.get(
                name='after_hours_contact_info')
            context['after_hours_contact_info'] = contact_info.value
        except Setting.DoesNotExist:
            pass

        try:
            instances = Instance.objects.filter(end=None, send_terminate=False)
        except Instance.DoesNotExist:
            instances = None

        context['instances'] = instances

        upcoming_emails = Email.objects.sending_this_week()
        context['upcoming_emails'] = upcoming_emails

        return context

##
# Emails
##
class EmailListView(EmailsMixin, SortSearchMixin, ListView):
    model = Email
    template_name = 'manager/emails.html'
    context_object_name = 'emails'
    paginate_by = 20

    def get_queryset(self):
        self.search_field = 'title'
        self.search_form = EmailSearchForm(self.request.GET)
        emails = super(EmailListView, self).get_queryset()
        return emails

    def get_context_data(self, **kwargs):
        context = super(EmailListView, self).get_context_data(**kwargs)
        context['search_form'] = self.search_form
        context['search_valid'] = self._search_valid
        return context

class EmailCreateView(EmailsMixin, CreateView):
    model = Email
    template_name = 'manager/email-create.html'
    form_class = EmailCreateUpdateForm

    def form_valid(self, form):
        email = form.instance
        form.instance.creator = self.request.user

        # Enable override send in case the service interval misses the email
        now = datetime.now()
        if email.is_sending_today(now) and email.send_time >= now.time():
            email.send_override = True
        else:
            email.send_override = False

        messages.success(self.request, 'Email successfully created.')
        return super(EmailCreateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-email-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class EmailUpdateView(EmailsMixin, UpdateView):
    model = Email
    template_name = 'manager/email-update.html'
    form_class = EmailCreateUpdateForm

    def get_form(self, form_class=None):
        form = super(EmailUpdateView, self).get_form(form_class)
        email = form.instance
        if email is not None:
            selected_recipient_groups = email.recipient_groups.all()
            active_recipient_groups = form.fields['recipient_groups'].queryset
            # Always show all selected recipient groups for the email,
            # even if the group(s) have been archived:
            valid_recipient_groups = (active_recipient_groups | selected_recipient_groups).distinct()
            form.fields['recipient_groups'].queryset = valid_recipient_groups
        return form

    def form_valid(self, form):
        email = form.instance

        # Enable override send in case the service interval misses the email
        now = datetime.now()
        if 'start_date' in form.changed_data or \
                'send_time' in form.changed_data or \
                'recurrence' in form.changed_data:
            if email.is_sending_today(now) and email.send_time >= now.time():
                email.send_override = True
            else:
                email.send_override = False

            # Clear the estimated times
            email.preview_est_time = None
            email.live_est_time = None

        email.updated_at = now
        messages.success(self.request, 'Email successfully updated.')
        return super(EmailUpdateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-email-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class EmailPlaceholderVerificationView(EmailsMixin, DetailView):
    model = Email
    template_name = 'manager/email-placeholder-verification.html'

    def get_context_data(self, **kwargs):
        context = super(EmailPlaceholderVerificationView, self).get_context_data(**kwargs)
        placeholders = self.object.placeholders
        context['attributes'] = []
        for placeholder in placeholders:
            emails = self.object.recipients.exclude(attributes__name=placeholder)
            if len(emails) > 0:
                context['attributes'].append((placeholder, emails[:10], len(emails)))
        return context

class EmailInstantSendView(EmailsMixin, FormView):
    template_name = 'manager/email-instant-send.html'
    form_class = EmailInstantSendForm

    def get_initial(self):
        initial = super(EmailInstantSendView, self).get_initial()

        if 'email_id' in self.request.GET:
            email = Email.objects.get(pk=self.request.GET.get('email_id'))
            initial['subject'] = "**TEST** " + email.subject + " **TEST**"
            initial['source_html_uri'] = email.source_html_uri
            initial['from_email_address'] = email.from_email_address
            initial['from_friendly_name'] = email.from_friendly_name
            initial['replace_delimiter'] = email.replace_delimiter

        return initial

    def form_valid(self, form):
        subject = form.cleaned_data['subject']
        source_html_uri = form.cleaned_data['source_html_uri']
        from_email_address = form.cleaned_data['from_email_address']
        from_friendly_name = form.cleaned_data['from_friendly_name']
        replace_delimiter = form.cleaned_data['replace_delimiter']
        recipient_groups = form.cleaned_data['recipient_groups']

        email = Email()
        email.subject = subject
        email.source_html_uri = source_html_uri
        email.from_email_address = from_email_address
        email.from_friendly_name = from_friendly_name
        email.replace_delimiter = replace_delimiter
        email.preview = False

        recipients = []
        for recipient_group in recipient_groups.all():
            for recipient in recipient_group.recipients.all():
                recipients.append(recipient)

        sender = EmailSender(email, recipients)

        try:
            sender.send()
        except Exception as e:
            form._errors['__all__'] = ErrorList([str(e)])
            return super(EmailInstantSendView, self).form_invalid(form)
        else:
            messages.success(self.request, 'Emails successfully sent.')
            return super(EmailInstantSendView, self).form_valid(form)

    def get_success_url(self):
        messages.success(self.request, 'Email sent')
        return reverse('manager-emails')

class EmailDeleteView(EmailsMixin, DeleteView):
    model = Email
    template_name = 'manager/email-delete-confirm.html'
    template_name_suffix = '-delete-confirm'

    def get_success_url(self):
        messages.success(self.request, 'Email successfully deleted.')
        return reverse('manager-emails')

class EmailUnsubscriptionsListView(EmailsMixin, ListView):
    model = Recipient
    template_name = 'manager/email-unsubscriptions.html'
    context_object_name = 'recipients'
    paginate_by = 20

    def dispatch(self, request, *args, **kwargs):
        self._email = get_object_or_404(Email, pk=kwargs['pk'])
        return super(EmailUnsubscriptionsListView, self).dispatch(request, *args, **kwargs)

    def get_queryset(self):
        return self._email.unsubscriptions.all()

    def get_context_data(self, **kwargs):
        context = super(EmailUnsubscriptionsListView,
                        self).get_context_data(**kwargs)
        context['email'] = self._email
        return context

class PreviewInstanceListView(EmailsMixin, ListView):
    model = PreviewInstance
    template_name = 'manager/email-preview-instances.html'
    paginate_by = 20
    context_object_name = 'preview_instances'

    def dispatch(self, request, *args, **kwargs):
        self._email = get_object_or_404(Email, pk=kwargs['pk'])
        return super(PreviewInstanceListView, self).dispatch(request,
                                                             *args,
                                                             **kwargs)

    def get_queryset(self):
        return PreviewInstance.objects.filter(email=self._email)

    def get_context_data(self, **kwargs):
        context = super(PreviewInstanceListView, self). \
            get_context_data(**kwargs)
        context['email'] = self._email
        return context

class LockContentView(UpdateView):
    model = PreviewInstance
    form_class = PreviewInstanceLockForm

    def get_success_url(self):
        """
        Return the user to the preview instances list for the parent email
        """
        # theobject = self.object.lock_content
        # blah = self.object.pk
        # raise Exception
        return reverse('manager-email-preview-instances',
                       args=(self.object.email.pk,))

class InstanceListView(EmailsMixin, ListView):
    model = Instance
    template_name = 'manager/email-instances.html'
    paginate_by = 20
    context_object_name = 'instances'

    def dispatch(self, request, *args, **kwargs):
        self._email = get_object_or_404(Email, pk=kwargs['pk'])
        return super(InstanceListView, self).dispatch(request, *args, **kwargs)

    def get_queryset(self):
        return Instance.objects.filter(email=self._email)

    def get_context_data(self, **kwargs):
        context = super(InstanceListView, self).get_context_data(**kwargs)
        context['email'] = self._email
        return context

class InstanceDetailView(EmailsMixin, DetailView):
    model = Instance
    template_name = 'manager/email-instance.html'
    context_object_name = 'instance'

def instance_cancel(request, pk):
    retval= {}

    # get the instance id
    if request.POST:
        instance_id = request.POST['email-instance-id']
    else:
        instance_id = pk

    if instance_id:
        try:
            instance = Instance.objects.get(pk=instance_id)

            if not instance.end:
                instance.send_terminate = True
                instance.save()
                retval['message'] = "Email instance send_terminate set to true."
            else:
                retval['message'] = "Email instance send has already ended, set_terminate left unchanged."

            retval['success'] = True
        except Instance.DoesNotExist:
            retval['success'] = False
            retval['error'] = {}
            retval['error']['code'] = 404
            retval['error']['message'] = "Email instance does not exist"

    return HttpResponse(json.dumps(retval), content_type='application/json')

##
# Recipients Groups
##
class RecipientGroupBaseListView(RecipientGroupsMixin, SortSearchMixin, ListView):
    model = RecipientGroup
    template_name = 'manager/recipientgroups.html'
    context_object_name = 'groups'
    paginate_by = 20

    def get_queryset(self):
        self.search_field = 'name'
        self.search_form = RecipientGroupSearchForm(self.request.GET)
        recipient_groups = super(RecipientGroupBaseListView, self).get_queryset()
        if not self.request.GET.get('status') or self.request.GET.get('status') == 'Active':
            recipient_groups = recipient_groups.filter(archived=False)
        elif self.request.GET.get('status') == 'Archived':
            recipient_groups = recipient_groups.filter(archived=True)
        return recipient_groups

    def get_context_data(self, **kwargs):
        context = super(RecipientGroupBaseListView, self).get_context_data(**kwargs)
        context['search_form'] = self.search_form
        context['search_valid'] = self._search_valid
        context['status'] = 'Active' if not self.request.GET.get(
            'status') else self.request.GET.get(
            'status')
        return context

class RecipientGroupLiveListView(RecipientGroupBaseListView):
    def get_queryset(self):
        recipient_groups = super(RecipientGroupLiveListView, self).get_queryset()
        recipient_groups = recipient_groups.filter(preview=False)
        return recipient_groups

    def get_context_data(self, **kwargs):
        context = super(RecipientGroupLiveListView,
                        self).get_context_data(**kwargs)
        context['preview'] = False
        return context

class RecipientGroupPreviewListView(RecipientGroupBaseListView):
    def get_queryset(self):
        recipient_groups = super(RecipientGroupPreviewListView, self).get_queryset()
        recipient_groups = recipient_groups.filter(preview=True)
        return recipient_groups

    def get_context_data(self, **kwargs):
        context = super(RecipientGroupPreviewListView,
                        self).get_context_data(**kwargs)
        context['preview'] = True
        return context

class RecipientGroupCreateView(RecipientGroupsMixin, CreateView):
    model = RecipientGroup
    template_name = 'manager/recipientgroup-create.html'
    form_class = RecipientGroupCreateForm

    def form_valid(self, form):
        messages.success(self.request, 'Recipient group successfully created.')
        return super(RecipientGroupCreateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-recipientgroup-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class RecipientGroupUpdateView(RecipientGroupsMixin, UpdateView):
    model = RecipientGroup
    template_name = 'manager/recipientgroup-update.html'
    form_class = RecipientGroupUpdateForm

    def get_context_data(self, **kwargs):
        context = super(RecipientGroupUpdateView, self).get_context_data(**kwargs)
        recipient_group = RecipientGroup.objects.get(pk=self.object.pk)
        recipients = recipient_group.recipients.all()
        paginator = Paginator(recipients, 20)

        page = self.request.GET.get('page')
        try:
            context['recipients'] = paginator.page(page)
        except PageNotAnInteger:
            context['recipients'] = paginator.page(1)
        except EmptyPage:
            context['recipients'] = paginator.page(paginator.num_pages)

        return context

    def form_valid(self, form):
        recipient_email = self.request.POST.get('recipient-email')

        if recipient_email:
            recipient_group = RecipientGroup.objects.get(pk=self.object.pk)
            try:
                recipient = Recipient.objects.get(email_address=recipient_email)
                if recipient not in recipient_group.recipients.all():
                    recipient_group.recipients.add(recipient)
                    recipient_group.save()
                else:
                    messages.warning(self.request, 'Recipient %s already in %s.' % (recipient_email, recipient_group.name))
            except Recipient.DoesNotExist:
                recipient = Recipient(email_address=recipient_email)
                recipient.save()
                recipient_group.recipients.add(recipient)
                recipient_group.save()
                messages.success(self.request, 'Recipient %s successfully created and added to %s.' % (recipient_email, recipient_group.name))
                return super(RecipientGroupUpdateView, self).form_valid(form)
            except Exception as e:
                message.errors(self.request, 'An error occurred: %s', e.strerror)
                return super(RecipientGroupUpdateView, self).form_invalid(form)

        messages.success(self.request, 'Recipient group successfully updated.')
        return super(RecipientGroupUpdateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-recipientgroup-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class RecipientGroupDeleteView(RecipientGroupsMixin, DeleteView):
    model = RecipientGroup
    template_name = 'manager/recipientgroup-delete-confirm.html'

    def get_context_data(self, **kwargs):
        context = super(RecipientGroupDeleteView, self).get_context_data(**kwargs)
        recurring_emails = Email.objects.filter(active=True, recipient_groups__id=self.object.pk).exclude(recurrence=0)
        upcoming_emails = Email.objects.filter(active=True, start_date__gt=datetime.now(), recurrence=0)
        context['recurring_emails'] = recurring_emails
        context['upcoming_emails'] = upcoming_emails
        return context

    def get_success_url(self):
        return reverse('manager-recipientgroups')

##
# Segments
##

class SegmentListView(ListView):
    model = Segment
    template_name = 'manager/segments-list.html'
    context_object_name = 'segments'
    paginate_by = 20

class SegmentCreateView(CreateView):
    model = Segment
    form_class = SegmentForm
    template_name = 'manager/segments-create.html'

    def get_context_data(self, **kwargs):
        data = super(SegmentCreateView, self).get_context_data(**kwargs)
        if self.request.POST:
            data['include_rules'] = IncludeSegmentRuleFormset(
                self.request.POST,
                prefix='include_rules'
            )
            data['exclude_rules'] = ExcludeSegmentRuleFormset(
                self.request.POST,
                prefix='exclude_rules'
            )
        else:
            data['include_rules'] = IncludeSegmentRuleFormset(
                prefix='include_rules'
            )
            data['exclude_rules'] = ExcludeSegmentRuleFormset(
                prefix='exclude_rules'
            )

        return data

    def form_valid(self, form):
        self.object = form.save()
        context = self.get_context_data(form=form)
        include_formset = context['include_rules']
        exclude_formset = context['exclude_rules']

        if include_formset.is_valid() and exclude_formset.is_valid():
            response = super(SegmentCreateView, self).form_valid(form)
            for idx, subform in enumerate(include_formset.forms):
                cleaned_data = subform.cleaned_data
                if cleaned_data:
                    rule = SegmentRule(
                        segment=self.object,
                        rule_type='include',
                        field=cleaned_data['field'],
                        conditional='AND' if idx == 0 else cleaned_data['conditional'],
                        key=None if cleaned_data['field'] not in ['has_attribute', 'clicked_url_in_instance'] else cleaned_data['key'],
                        value=cleaned_data['value'],
                        index=idx
                    )
                    rule.save()

            for idx, subform in enumerate(exclude_formset.forms):
                cleaned_data = subform.cleaned_data
                if cleaned_data:
                    rule = SegmentRule(
                        segment=self.object,
                        rule_type='exclude',
                        field=cleaned_data['field'],
                        conditional='AND' if idx == 0 else cleaned_data['conditional'],
                        key=None if cleaned_data['field'] not in ['has_attribute', 'clicked_url_in_instance'] else cleaned_data['key'],
                        value=cleaned_data['value'],
                        index=idx
                    )
                    rule.save()

            return response
        else:
            return super(SegmentCreateView, self).form_invalid(form)

    def get_success_url(self):
        return reverse('manager-segments-update', args=(self.object.id,))

class SegmentUpdateView(UpdateView):
    model = Segment
    form_class = SegmentForm
    template_name = 'manager/segments-update.html'

    def get_context_data(self, **kwargs):
        data = super(SegmentUpdateView, self).get_context_data(**kwargs)
        if self.request.POST:
            data['include_rules'] = IncludeSegmentRuleFormset(
                self.request.POST,
                instance=self.object,
                prefix='include_rules'
            )
            data['exclude_rules'] = ExcludeSegmentRuleFormset(
                self.request.POST,
                instance=self.object,
                prefix='exclude_rules'
            )
        else:
            data['include_rules'] = IncludeSegmentRuleFormset(
                instance=self.object,
                prefix='include_rules',
                queryset=self.object.include_rules.all()
            )
            data['exclude_rules'] = ExcludeSegmentRuleFormset(
                instance=self.object,
                prefix='exclude_rules',
                queryset=self.object.exclude_rules.all()
            )

        return data

    def form_valid(self, form):
        self.object = form.save(commit=False)
        context = self.get_context_data(form=form)
        include_formset = context['include_rules']
        exclude_formset = context['exclude_rules']

        if include_formset.is_valid() and exclude_formset.is_valid():
            self.object.include_rules.delete()
            self.object.exclude_rules.delete()

            for idx, subform in enumerate(include_formset.ordered_forms):
                cleaned_data = subform.cleaned_data
                if cleaned_data:
                    rule = SegmentRule(
                        segment=self.object,
                        rule_type='include',
                        field=cleaned_data['field'],
                        conditional='AND' if idx == 0 else cleaned_data['conditional'],
                        key=None if cleaned_data['field'] not in ['has_attribute', 'clicked_url_in_instance'] else cleaned_data['key'],
                        value=cleaned_data['value'],
                        index=idx
                    )
                    rule.save()

            for idx, subform in enumerate(exclude_formset.ordered_forms):
                cleaned_data = subform.cleaned_data
                if cleaned_data:
                    rule = SegmentRule(
                        segment=self.object,
                        rule_type='exclude',
                        field=cleaned_data['field'],
                        conditional='AND' if idx == 0 else cleaned_data['conditional'],
                        key=None if cleaned_data['field'] not in ['has_attribute', 'clicked_url_in_instance'] else cleaned_data['key'],
                        value=cleaned_data['value'],
                        index=idx
                    )
                    rule.save()

            form.save()
            response = super(SegmentUpdateView, self).form_valid(form)

            return response
        else:
            return super(SegmentUpdateView, self).form_invalid(form)

    def get_success_url(self):
        return reverse('manager-segments-update', args=(self.object.id,))

class SegmentDeleteView(DeleteView):
    model = Segment
    template_name = 'manager/segments-delete.html'

    def get_success_url(self):
        return reverse('manager-segments')

##
# Recipients
##
class RecipientListView(RecipientsMixin, SortSearchMixin, ListView):
    model = Recipient
    template_name = 'manager/recipients.html'
    context_object_name = 'recipients'
    paginate_by = 20

    def get_queryset(self):
        self.search_field = 'email_address'
        self.search_form = EmailSearchForm(self.request.GET)
        super(RecipientListView, self).get_queryset()

        self._search_form = RecipientSearchForm(self.request.GET)
        self._search_valid = self._search_form.is_valid()
        if self._search_valid:
            return Recipient.objects.filter(email_address__icontains=self._search_form.cleaned_data['search_query'])
        else:
            return Recipient.objects.all().order_by('disable', 'email_address')

    def get_context_data(self, **kwargs):
        context = super(RecipientListView, self).get_context_data(**kwargs)
        context['search_form'] = self._search_form
        context['search_valid'] = self._search_valid
        context['search_query'] = self._search_form.cleaned_data['search_query'] if self._search_valid else ''
        return context

class RecipientCreateView(RecipientsMixin, CreateView):
    model = Recipient
    template_name = 'manager/recipient-create.html'
    context_object_name = 'recipient'
    form_class = RecipientCreateUpdateForm

    def get(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        recipient_attributes_formset = RecipientAttributeFormSet()
        return self.render_to_response(
            self.get_context_data(form=form, recipient_attributes_formset=recipient_attributes_formset))

    def post(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        recipient_attributes_formset = RecipientAttributeFormSet(self.request.POST)
        if (form.is_valid() and recipient_attributes_formset.is_valid()):
            return self.form_valid(form, recipient_attributes_formset)
        else:
            return self.form_invalid(form, recipient_attributes_formset)

    def form_valid(self, form, recipient_attributes_formset):
        self.object = form.save()
        recipient_attributes_formset.instance = self.object
        recipient_attributes_formset.save()
        messages.success(self.request, 'Recipient successfully created.')
        self.object.set_groups(form.cleaned_data['groups'])
        return super(RecipientCreateView, self).form_valid(form)

    def form_invalid(self, form, recipient_attributes_formset):
        messages.error(self.request, 'Please review the errors below and try again.')
        return self.render_to_response(
            self.get_context_data(form=form, recipient_attributes_formset=recipient_attributes_formset))

    def get_success_url(self):
        return reverse('manager-recipient-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class RecipientUpdateView(RecipientsMixin, UpdateView):
    model = Recipient
    template_name = 'manager/recipient-update.html'
    form_class = RecipientCreateUpdateForm
    form_attributes_class = RecipientAttributeCreateForm

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        recipient_attributes_formset = RecipientAttributeFormSet(instance=self.object)
        return self.render_to_response(
            self.get_context_data(form=form, recipient_attributes_formset=recipient_attributes_formset))

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        recipient_attributes_formset = RecipientAttributeFormSet(self.request.POST, instance=self.object)
        if (form.is_valid() and recipient_attributes_formset.is_valid()):
            return self.form_valid(form, recipient_attributes_formset)
        else:
            return self.form_invalid(form, recipient_attributes_formset)

    def form_valid(self, form, recipient_attributes_formset):
        self.object = form.save()
        recipient_attributes_formset.instance = self.object
        recipient_attributes_formset.save()
        messages.success(self.request, 'Recipient successfully updated.')
        self.object.set_groups(form.cleaned_data['groups'])
        return super(RecipientUpdateView, self).form_valid(form)

    def form_invalid(self, form, recipient_attributes_formset):
        messages.error(self.request, 'Please review the errors below and try again.')
        return self.render_to_response(
            self.get_context_data(form=form, recipient_attributes_formset=recipient_attributes_formset))

    def get_success_url(self):
        return reverse('manager-recipient-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class RecipientAttributeListView(RecipientsMixin, ListView):
    model = RecipientAttribute
    template_name = 'manager/recipient-recipientattributes.html'
    context_object_name = 'attributes'
    pageinate_by = 20

    def dispatch(self, request, *args, **kwargs):
        self._recipient = get_object_or_404(Recipient, pk=kwargs['pk'])
        return super(RecipientAttributeListView, self).dispatch(request,
                                                                *args,
                                                                **kwargs)

    def get_queryset(self):
        return RecipientAttribute.objects.filter(recipient=self._recipient)

    def get_context_data(self, **kwargs):
        context = super(RecipientAttributeListView,
                        self).get_context_data(**kwargs)
        context['recipient'] = self._recipient
        return context

class RecipientAttributeCreateView(RecipientsMixin, CreateView):
    model = RecipientAttribute
    template_name = 'manager/recipientattribute-create.html'
    form_class = RecipientAttributeCreateForm

    def dispatch(self, request, *args, **kwargs):
        self._recipient = get_object_or_404(Recipient, pk=kwargs['pk'])
        return super(RecipientAttributeCreateView, self).dispatch(request,
                                                                  *args,
                                                                  **kwargs)

    def get_context_data(self, **kwargs):
        context = super(RecipientAttributeCreateView,
                        self).get_context_data(**kwargs)
        context['recipient'] = self._recipient
        return context

    def form_valid(self, form):
        form.instance.recipient = self._recipient

        # Check the unique_together(recipient, name) here. It can't be done in
        # the Form class
        name = form.cleaned_data['name']
        try:
            RecipientAttribute.objects.get(name=name,
                                           recipient=self._recipient)
        except RecipientAttribute.DoesNotExist:
            pass
        else:
            form._errors['name'] = ErrorList(['A attribute with that name already exists for this recipient.'])
            return super(RecipientAttributeCreateView, self).form_invalid(form)

        return super(RecipientAttributeCreateView, self).form_valid(form)

    def get_success_url(self):
        messages.success(self.request,
                         'Recipient attribute successfully created.')
        return reverse('manager-recipient-recipientattributes',
                       args=(),
                       kwargs={'pk': self._recipient.pk})

class RecipientAttributeUpdateView(RecipientsMixin, UpdateView):
    model = RecipientAttribute
    template_name = 'manager/recipientattribute-update.html'
    form_class = RecipientAttributeUpdateForm
    context_object_name = 'attribute'

    def get_success_url(self):
        messages.success(self.request,
                         'Recipient attribute successfully updated.')
        return reverse('manager-recipientattribute-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class RecipientAttributeDeleteView(RecipientsMixin, DeleteView):
    model = RecipientAttribute
    template_name = 'manager/recipientattribute-delete.html'
    template_name_suffix = '-delete-confirm'
    context_object_name = 'attribute'

    def get_success_url(self):
        messages.success(self.request, 'Attribute successfully deleted.')
        return reverse('manager-recipient-recipientattributes',
                       args=(),
                       kwargs={'pk': self.object.recipient.pk})

class RecipientSubscriptionsUpdateView(UpdateView):
    model = Recipient
    template_name = 'manager/recipient-subscriptions.html'
    form_class = RecipientSubscriptionsForm

    def get_context_data(self, **kwargs):
        context = super(RecipientSubscriptionsUpdateView, self).get_context_data()
        context['subscription_categories'] = SubscriptionCategory.objects.all()
        return context

    def get_object(self, *args, **kwargs):
        mac = self.request.GET.get('mac', None)

        recipient_id = self.request.GET.get('recipient', None)
        email_id = self.request.GET.get('email', None)

        # Old style unsubscribe
        if recipient_id is not None and email_id is not None:
            try:
                recipient = Recipient.objects.get(pk=recipient_id)
            except Recipient.DoesNotExist:
                raise PermissionDenied
            if mac is None or mac != calc_unsubscribe_mac_old(recipient_id, email_id):
                raise PermissionDenied
        else:
            recipient = super(RecipientSubscriptionsUpdateView, self).get_object()
            # Validate MAC
            if mac is None or mac != calc_unsubscribe_mac(recipient.pk):
                raise PermissionDenied
        return recipient

    def form_valid(self, form):
        subscriptions = form.cleaned_data['subscription_categories']
        unsubscriptions = list(set(SubscriptionCategory.objects.all()) - set(subscriptions))

        for category in unsubscriptions:
            category.unsubscriptions.add(self.object)

        for category in subscriptions:
            category.unsubscriptions.remove(self.object)

        return super(RecipientSubscriptionsUpdateView, self).form_valid(form)

    def get_success_url(self):
        messages.success(self.request,
                         'Your subscriptions have been successfully updated.')
        return self.object.unsubscribe_url

class CampaignListView(ListView):
    model = Campaign
    template_name = 'manager/campaign-list.html'
    context_object_name = 'campaigns'

class CampaignCreateView(CreateView):
    model = Campaign
    template_name = 'manager/campaign-create.html'
    form_class = CampaignForm

    def form_valid(self, form):
        messages.success(self.request, 'Campaign successfully created.')
        return super(CampaignCreateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-campaigns-update',
                      args=(),
                      kwargs={'pk': self.object.pk})

class CampaignUpdateView(UpdateView):
    model = Campaign
    template_name = 'manager/campaign-update.html'
    form_class = CampaignForm

    def form_valid(self, form):
        messages.success(self.request, 'Campaign successfully updated.')
        return super(CampaignUpdateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-campaigns-update',
                      args=(),
                      kwargs={'pk': self.object.pk})

class CampaignDeleteView(DeleteView):
    model = Campaign
    template_name = 'manager/campaign-delete.html'
    context_object_name = 'campaign'

    def get_success_url(self):
        messages.success(self.request, 'Campaign successfully deleted.')
        return reverse('manager-campaigns',
                        args = (),
                        kwargs={})

class CampaignStatView(DetailView):
    model = Campaign
    template_name = 'manager/campaign-stats.html'
    context_object_name = 'campaign'

    def get_context_data(self, **kwargs):
        ctx = super().get_context_data(**kwargs)
        ctx['instance_count'] = 0
        campaign = ctx.get('campaign', None)
        if campaign:
            for email in campaign.emails.all():
                ctx['instance_count'] += email.instances.count()

        return ctx

class SubscriptionCategoryListView(SubscriptionsMixin, ListView):
    model = SubscriptionCategory
    template_name = 'manager/subscription-category-list.html'
    context_object_name = 'categories'

class SubscriptionCategoryCreateView(SubscriptionsMixin, CreateView):
    model = SubscriptionCategory
    template_name = 'manager/subscription-category-create.html'
    form_class = SubscriptionCategoryForm

    def form_valid(self, form):
        messages.success(self.request, 'Subscription Category successfully created.')
        return super(SubscriptionCategoryCreateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-subscription-categories-update',
                      args=(),
                      kwargs={'pk': self.object.pk})

class SubscriptionCategoryUpdateView(SubscriptionsMixin, UpdateView):
    model= SubscriptionCategory
    template_name = 'manager/subscription-category-update.html'
    form_class = SubscriptionCategoryForm

    def form_valid(self, form):
        messages.success(self.request, 'Subscription Category successfully updated.')
        return super(SubscriptionCategoryUpdateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-subscription-categories-update',
                      args=(),
                      kwargs={'pk': self.object.pk})

class SettingListView(SettingsMixin, ListView):
    model = Setting
    template_name = 'manager/settings.html'
    context_object_name = 'settings'
    pageinate_by = 20

class SettingCreateView(SettingsMixin, CreateView):
    model = Setting
    template_name = 'manager/setting-create.html'
    form_class = SettingCreateUpdateForm

    def form_valid(self, form):
        messages.success(self.request, 'Setting successfully created.')
        return super(SettingCreateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-setting-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class SettingUpdateView(SettingsMixin, UpdateView):
    model = Setting
    template_name = 'manager/setting-update.html'
    form_class = SettingCreateUpdateForm

    def form_valid(self, form):
        messages.success(self.request, 'Setting successfully updated.')
        return super(SettingUpdateView, self).form_valid(form)

    def get_success_url(self):
        return reverse('manager-setting-update',
                       args=(),
                       kwargs={'pk': self.object.pk})

class SettingDeleteView(SettingsMixin, DeleteView):
    model = Setting
    template_name = 'manager/setting-delete-confirm.html'
    template_name_suffix = '-delete-confirm'

    def get_success_url(self):
        messages.success(self.request, 'Setting successfully deleted.')
        return reverse('manager-settings')

##
# Tracking
##
def redirect(request):
    '''
        Redirects based on URL and records URL click
    '''
    instance_id = request.GET.get('instance', None)
    url_string = request.GET.get('url', None)
    position = request.GET.get('position', None)
    recipient_id = request.GET.get('recipient', None)
    mac = request.GET.get('mac', None)
    parser = HTMLParser()

    if not url_string or not position or not recipient_id or not mac or not instance_id:
        raise Http404("Poll does not exist")
    else:
        url_string = urllib.parse.unquote(url_string)
        url_part    = url_string.split('?')[0]

        if not URL.objects.filter(name__startswith=url_part).exists():
            raise Http404("Poll does not exist")
        # No matter what happens, make sure the redirection works
        try:
            if position and recipient_id and mac and instance_id:
                try:
                    position = int(position)
                    recipient_id = int(recipient_id)
                    instance_id = int(instance_id)
                except ValueError:
                    log.error('value error')
                    pass
                else:
                    if mac == calc_url_mac(url_string,
                                           position,
                                           recipient_id,
                                           instance_id):
                        try:
                            recipient = Recipient.objects.get(id=recipient_id)
                            instance = Instance.objects.get(id=instance_id)
                            url = URL.objects.get(name=url_string,
                                                  position=position,
                                                  instance=instance)
                        except URL.DoesNotExist:
                            # This should have been created
                            # in the mailer-process command
                            # when this email was sent
                            pass
                        except Recipient.DoesNotExist:
                            # strange
                            log.error('bad recipient')
                            pass
                        except Instance.DoesNotExist:
                            # also strange
                            log.error('bad instance')
                            pass
                        else:
                            url_click, created = URLClick.objects.get_or_create(recipient=recipient, url=url)
                            log.debug('url click saved')
                    else:
                        log.error('wrong mac')
            else:
                log.error('something none')
        except Exception as e:
            log.error(str(e))
            pass
        # Decode any encoded characters to ensure things like
        # UTM params work appropriately

        url_string = parser.unescape(url_string)
        return HttpResponseRedirect(url_string)

def instance_open(request):
    '''
        Records an email open
    '''
    instance_id = request.GET.get('instance', None)
    recipient_id = request.GET.get('recipient', None)
    mac = request.GET.get('mac', None)

    if recipient_id and mac and instance_id is not None:
        try:
            instance_id = int(instance_id)
            recipient_id = int(recipient_id)
        except ValueError:
            # corrupted
            pass
        else:
            if mac == calc_open_mac(recipient_id, instance_id):
                try:
                    recipient = Recipient.objects.get(id=recipient_id)
                    instance = Instance.objects.get(id=instance_id)

                    instance_open, created = InstanceOpen.objects.get_or_create(recipient=recipient, instance=instance, is_reopen=False)
                    if not created:
                        instance_new = InstanceOpen(recipient=recipient, instance=instance, is_reopen=True)
                        instance_new.save()
                        log.debug('re-open created')
                    else:
                        log.debug('open created')
                except InstanceOpen.MultipleObjectsReturned:
                    log.error('multiple InstanceOpens returned');
                    pass
                except Recipient.DoesNotExist:
                    # strange
                    log.error('bad recipient')
                    pass
                except Instance.DoesNotExist:
                    # also strange
                    log.error('bad instance')
                    pass
    return HttpResponse(settings.DOT, content_type='image/png')

##
# Utility Views
##
class RecipientCSVImportView(RecipientsMixin, FormView):
    template_name = 'manager/recipient-csv-import.html'
    form_class = RecipientCSVImportForm

    def form_valid(self, form):
        if form.is_valid():
            existing_group_name = None

            if form.cleaned_data['existing_group_name']:
                existing_group_name = form.cleaned_data['existing_group_name'].name

            new_group_name = form.cleaned_data['new_group_name']
            columns = list(col.strip() for col in form.cleaned_data['column_order'].split(','))
            column_order = ','.join(columns)
            skip_first_row = form.cleaned_data['skip_first_row']
            remove_stale = form.cleaned_data['remove_stale']
            csv_file = form.cleaned_data['csv_file']
            existing_group_id = None

            group = ""
            if existing_group_name is not None:
                group = existing_group_name
                existing_group_id = form.cleaned_data['existing_group_name'].pk
            else:
                group = new_group_name

            filename = settings.UPLOAD_DIR + "/" +  csv_file.name

            with open(filename, 'wb+') as dest:
                for chunk in csv_file.chunks():
                    dest.write(chunk)

            if (existing_group_id):
                success_url = reverse('manager-recipientgroup-update', kwargs={'pk': existing_group_id})
            else:
                success_url = None

            tracker = SubprocessStatus.objects.create(
                name="Importing {0} into \"{1}\"...".format(csv_file.name, group),
                current_unit=1,
                total_units=1,
                unit_name='recipients',
                success_url=success_url,
                back_url=reverse('manager-recipients-csv-import')
            )

            self.tracker_pk = tracker.pk

            command = [
                sys.executable,
                'manage.py',
                'import-emails',
                filename,
                '--group-name={0}'.format(group),
                '--column-order={0}'.format(column_order),
                '--subprocess={0}'.format(self.tracker_pk)
            ]

            if skip_first_row:
                command.append('--ignore-first-row')

            if remove_stale:
                command.append('--remove-stale')

            subprocess.Popen(
                command,
                close_fds=True,
                cwd=settings.BASE_DIR
            )

            # messages.success(self.request, 'Emails successfully imported.')
            return super(RecipientCSVImportView, self).form_valid(form)

    def get_success_url(self):
        return reverse('subprocess-status-detail-view', kwargs={ 'pk': self.tracker_pk })

class ExportCleanupView(FormView):
    template_name = 'manager/export-cleanup.html'
    form_class = ExportCleanupForm

    def get_context_data(self, **kwargs):
        context = super(ExportCleanupView, self).get_context_data(**kwargs)

        removal_hash = self.request.GET.get('hash', None)
        context['hash'] = removal_hash

        if removal_hash:
            try:
                stale = StaleRecord.objects.get(removal_hash=removal_hash)
                earliest = stale.instances.aggregate(Min('end'))
                latest = stale.instances.aggregate(Max('end'))
                context['stale'] = stale
                context['earliest'] = earliest['end__min'] if 'end__min' in earliest else None
                context['latest'] = latest['end__max'] if 'end__max' in latest else None
            except StaleRecord.DoesNotExist:
                context['stale'] = None

        return context

    def form_valid(self, form):
        if form.is_valid:
            removal_hash = self.request.GET.get('hash')

            if 'remove_emails' in form.cleaned_data:
                remove_emails = form.cleaned_data['remove_emails']
            else:
                remove_emails = False

            success_url = reverse('manager-home')
            back_url = reverse('manager-export-cleanup')

            self.tracker = SubprocessStatus.objects.create(
                name="Deleting stale records...",
                current_unit=1,
                total_units=1,
                unit_name='records',
                success_url=success_url,
                back_url=back_url
            )

            command = [
                sys.executable,
                'manage.py',
                'remove-stale',
                removal_hash,
                '--quiet=True',
                '--subprocess={0}'.format(self.tracker.pk)
            ]

            if remove_emails:
                command.append('--remove-empty-emails=True')

            subprocess.Popen(
                command,
                close_fds=True,
                cwd=settings.BASE_DIR
            )

            return super(ExportCleanupView, self).form_valid(form)

    def get_success_url(self):
        return reverse('subprocess-status-detail-view', kwargs={ 'pk': self.tracker.pk })

class SubprocessStatusDetailView(DetailView):
    model = SubprocessStatus
    template_name = 'manager/subprocess-status.html'

def instance_json_feed(request):
    retval = {}

    def datetime_to_milliseconds(datetime_object):
        if datetime_object:
            return time.mktime(datetime_object.timetuple()) * 1000
        else:
            return 0

    if request.GET.get('pk'):
        pk = request.GET.get('pk')
        try:
            instance = Instance.objects.get(pk=pk)

            retval['sent_count'] = instance.sent_count
            retval['total'] = instance.recipient_details.count()
            retval['start'] = datetime_to_milliseconds(instance.start)
            retval['end'] = datetime_to_milliseconds(instance.end)
        except Instance.DoesNotExist:
            retval['error'] ='Error getting instance send information.'
            pass

    return HttpResponse(json.dumps(retval), content_type='application/json')

def subprocess_status_json_feed(request):
    retval = {}

    if request.GET.get('pk'):
        pk = request.GET.get('pk')

        try:
            sp = SubprocessStatus.objects.get(pk=pk)

            retval['current_unit'] = sp.current_unit
            retval['total_units'] = sp.total_units
            retval['unit_name'] = sp.unit_name
            retval['status'] = sp.status
            retval['error'] = sp.error
            retval['success_url'] = sp.success_url
            retval['back_url'] = sp.back_url
        except SubprocessStatus.DoesNotExist:
            retval['error'] = 'Error getting Subprocess Status'
            pass

    return HttpResponse(json.dumps(retval), content_type='application/json')

def recipient_json_feed(request):
    search_term = ''

    if request.GET.get('search'):
        search_term = request.GET.get('search')

    recipients = []

    if search_term:
        recipient_fks = list(RecipientAttribute.objects.filter(name__in=['First Name','Last Name','Preferred Name'], value__contains=search_term).values_list('recipient', flat=True).distinct())
        recipient_emails = list(Recipient.objects.filter(email_address__contains=search_term).values_list('pk', flat=True))

        for r in recipient_emails:
            if r not in recipient_fks:
                recipient_fks.append(r)

        for r in recipient_fks:
            recipients.append(Recipient.objects.get(pk=r))

    else:
        recipients = Recipient.objects.all()

    retval = []

    for recipient in recipients:
        first_name = RecipientAttribute.objects.filter(recipient=recipient, name='First Name')[0].value if RecipientAttribute.objects.filter(recipient=recipient, name='First Name') else ''
        last_name = RecipientAttribute.objects.filter(recipient=recipient, name='Last Name')[0].value if RecipientAttribute.objects.filter(recipient=recipient, name='Last Name') else ''
        preferred_name = RecipientAttribute.objects.filter(recipient=recipient, name='Preferred Name')[0].value if RecipientAttribute.objects.filter(recipient=recipient, name='Preferred Name') else ''

        r = {}
        r['pk'] = recipient.pk
        r['email_address'] = recipient.email_address
        r['first_name'] = first_name
        r['last_name'] = last_name
        r['preferred_name'] = preferred_name

        retval.append(r)

    return HttpResponse(json.dumps(retval), content_type='application/json')

def objects_as_options(request):
    """
    Provides a simple API endpoint for retrieving
    various postmaster objects as options for a
    select2 control.
    """
    object_type = request.GET.get('type', None)
    query = request.GET.get('q', None)

    retval = {}
    ret_status = 200

    if object_type:
        results = []

        if object_type == 'recipientgroup':
            objects = RecipientGroup.objects.all()
            if query:
                objects = objects.filter(name__contains=query)
            results = [{'text': x.name, 'id': x.id} for x in objects]
        elif object_type == 'recipientattribute':
            objects = RecipientAttribute.objects.values_list('name').distinct()
            results = [{'text': x[0], 'id': x[0]} for x in objects]
        elif object_type == 'instance':
            objects = Instance.objects.all()
            if query:
                objects = objects.filter(
                    Q(email__title__icontains=query) |
                    Q(subject__icontains=query)
                )
            results = [{'text': x.option_text, 'id': x.id} for x in objects]
        elif object_type == 'email':
            objects = Email.objects.all()
            if query:
                objects = objects.filter(title__icontains=query)
            results = [{'text': x.title, 'id': x.id} for x in objects]
        elif object_type == 'url':
            objects = URL.objects.all()
            if query:
                objects = objects.filter(name__iconatins=query)
            results = [{'text': x.name, 'id': x.id} for x in objects]

        retval['results'] = results
    else:
        ret_status = 400
        retval = {
            'error': 'You must specify an object type to query for by passing a "type" parameter.'
        }

    return HttpResponse(json.dumps(retval), content_type='application/json', status=ret_status)

##
# Creates a recipient group based on email opens.
# POST only
##
def create_recipient_group_action(request):
    action = request.POST.get('group-create-action')

    email_instance_id = request.POST.get('email-instance-id')
    email_instance = Instance.objects.get(pk=email_instance_id)

    if (action == 'opens'):
        return create_recipient_group_email_opens(request, email_instance)
    elif (action == 'unopens'):
        return create_recipient_group_email_unopens(request, email_instance)
    elif (action == 'no-clicks'):
        return create_recipient_group_no_clicks(request, email_instance)
    else:
        return HttpResponse(
            '<h1>Invalid Action</h1>',
            status=400
        )

def create_recipient_group_email_opens(request, email_instance):
    '''
    Creates a recipient group based on email opens.
    POST only
    '''
    recipients = email_instance.open_recipients

    recipient_group = RecipientGroup(name=email_instance.email.title + ' Recipient Group ' + datetime.now().strftime('%m-%d-%y %I:%M %p'))
    if RecipientGroup.objects.filter(name=recipient_group.name).count() > 0:
        recipient_group.name = recipient_group.name + '-1'

    recipient_group.save()

    recipient_group.recipients.add(*recipients)

    recipient_group.save()

    messages.success(request, 'Recipient group successfully created. Please remember to update the name to something unique.')
    return HttpResponseRedirect(
        reverse('manager-recipientgroup-update',
            args=(),
            kwargs={'pk': recipient_group.pk}
        )
    )

def create_recipient_group_email_unopens(request, email_instance):
    '''
    Creates a recipient group based on emails
    that did not open the email
    POST only
    '''
    recipients_pks = InstanceOpen.objects.filter(instance=email_instance).values_list('recipient__pk', flat=True)

    # Remove all the opens from the sent recipients
    recipients = email_instance.recipients.exclude(pk__in=recipients_pks)

    recipient_group = RecipientGroup(name=email_instance.email.title + ' Recipient Group - Unopens - ' + datetime.now().strftime('%m-%d-%y %I:%M %p'))
    if RecipientGroup.objects.filter(name=recipient_group.name).count() > 0:
        recipient_group.name = recipient_group.name + '-1'

    recipient_group.save()

    recipient_group.recipients.add(*recipients)

    recipient_group.save()

    messages.success(request, 'Recipient group successfully created. Please remember to update the name to something unique.')
    return HttpResponseRedirect(
        reverse('manager-recipientgroup-update',
            args=(),
            kwargs={'pk': recipient_group.pk}
        )
    )

def create_recipient_group_no_clicks(request, email_instance):
    '''
    Creates a recipient group with recipients
    who did not click on any links for a
    particular email instance.
    '''
    recipients = email_instance.recipients.all()

    recipient_clicks = URLClick.objects.filter(url__in=email_instance.urls.all()).values('recipient__id').distinct()
    recipient_clicks = Recipient.objects.filter(id__in=recipient_clicks)

    recipients = list(set(recipients) - set(recipient_clicks))

    recipient_group = RecipientGroup(name=email_instance.email.title + ' Recipient Group - No Clicks - ' + datetime.now().strftime('%m-%d-%y %I:%M %p'))
    if RecipientGroup.objects.filter(name=recipient_group.name).count() > 0:
        recipient_group.name = recipient_group.name + '-1'

    recipient_group.save()

    recipient_group.recipients.add(*recipients)

    recipient_group.save()

    messages.success(request, 'Recipient group successfully created. Please remember to update the name to something unique.')

    return HttpResponseRedirect(
        reverse('manager-recipientgroup-update',
            args=(),
            kwargs={'pk': recipient_group.pk}
        )
    )

def create_recipient_group_url_clicks(request):
    '''
        Creates a recipient group based on url clicks.
        POST only
    '''
    url_ids = request.POST.getlist('url-pks[]')
    url_clicks = []
    for url_id in url_ids:
        url_clicks.append(URLClick.objects.filter(url=url_id))

    recipient_group = RecipientGroup(name='URL Click Recipient Group - ' + datetime.now().strftime('%m-%d-%y %I:%M %p'))
    recipient_group.save()

    recipients = []

    for url_click in url_clicks:
        for click in url_click:
            recipients.append(click.recipient)

    recipient_group.recipients.add(*recipients)

    recipient_group.save()

    messages.success(request, 'Recipient group successfully created. Please remember to update the name to something unique.')
    return HttpResponseRedirect(
        reverse('manager-recipientgroup-update',
            args=(),
            kwargs={'pk': recipient_group.pk}
        )
    )

def csv_export_recipient_group(request, pk):
    try:
        group = RecipientGroup.objects.get(pk=pk)
    except RecipientGroup.DoesNotExist:
        return HttpResponse(
            '<h1>The recipient group does not exist.</h1>',
            status=400
        )

    filename = "{0}-export.csv".format(group.name)

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="{0}"'.format(filename)

    writer = csv.writer(response)

    writer.writerow(['email'])

    for recipient in group.recipients.all():
        writer.writerow([recipient.email_address])

    return response

def stale_record_action(request):
    removal_hash = request.GET.get('hash')

    try:
        record = StaleRecord.objects.get(removal_hash=removal_hash)
    except StaleRecord.DoesNotExist:
        return HttpResponse(
            '<h1>The recipient group does not exist.</h1>',
            status=400
        )

    command = [
        sys.executable,
        'manage.py',
        'remove-stale',
        record.removal_hash,
        '--remove-empty-emails=True',
        '--quiet=True'
    ]

    subprocess.Popen(
        command,
        close_fds=True,
        cwd=settings.BASE_DIR
    )

    messages.success(request, 'Records removed.')

    return HttpResponseRedirect(
        reverse('manager-home',
            args=()
        )
    )

def s3_upload_user_file(request):
    """
    Uploads a unique file to Amazon S3, using a key prefixed with the current
    year + month.

    Returns json containing uploaded file url ( {link: '...'} ) or error
    message ( {error: '...'} ).
    """
    today = datetime.today()
    if request.method == 'POST':
        response_data = {}
        file = request.FILES['file']
        file_prefix = str(today.year) + '/' + '{:02d}'.format(today.month) + '/'
        protocol = request.POST.get('protocol')
        extension_groupname = request.POST.get('extension_groupname')
        unique = request.POST.get('unique')
        s3 = AmazonS3Helper()

        if file_prefix is None:
            file_prefix = ''

        if protocol not in s3.valid_protocols:
            protocol = '//'

        # convert to bool
        if not unique:
            unique = False
        else:
            unique = True

        if file is None:
            response_data['error'] = 'File not set.'
        else:
            try:
                keyobj = s3.upload_file(
                    file=file,
                    unique=unique,
                    file_prefix=file_prefix,
                    extension_groupname=extension_groupname
                )
            except AmazonS3Helper.KeyCreateError as e:
                response_data['error'] = 'Failed to upload file.'
            except PermissionDenied:
                response_data['error'] = 'Cannot upload this type of file.'

            try:
                url = keyobj.generate_url(0, query_auth=False, force_http=True)
            except Exception as e:
                response_data['error'] = 'Failed to generate url for file.'

            if url:
                if protocol != 'http://':
                    url = url.replace('http://', protocol)
                response_data['link'] = url

        return HttpResponse(
            json.dumps(response_data),
            content_type='application/json'
        )
    else:
        return HttpResponseForbidden()


import calendar
import time

import pytest

from conftest import driver
from utilities import data_reader
from webpages.homePage import HomePage
from webpages.loginPage import LoginPage
from webpages.registrationPage import RegistrationPage

test_data = data_reader.excel_to_dict("Test_Data/registration.xls")

@pytest.mark.registration
@pytest.mark.parametrize("test_params", test_data)
def test_registration_1(test_params, ):
    """
        Positive registration flow
    """
    home_page = HomePage(driver)
    home_page.click_sign_in()
    login = LoginPage(driver)
    ts = calendar.timegm(time.gmtime())
    email = test_params["firstname"]+str(ts)+"@gmail.com"
    login.enter_email_for_registration(email)
    login.click_submit_create()
    register = RegistrationPage(driver)
    register.register_user(test_params)
    time.sleep(5)
    login.sign_out_from_account()

@pytest.mark.registration
@pytest.mark.parametrize("email, error_message", [("shahrukh@gmail.com", "An account using this email address has "
                                                                         "already been registered. Please enter a "
                                                                         "valid password or request a new one."),
                                                  ("shahrukhgmail.com", "Invalid email address.")])
def test_registration_2(email, error_message):
    """
    1. Entering a incorrect email
    2. Entering already used email
    """
    home_page = HomePage(driver)
    home_page.click_sign_in()
    login = LoginPage(driver)
    login.enter_email_for_registration(email)
    login.click_submit_create()
    login.assert_text(login.email_create_error, error_message)


#encoding:utf-8
import torch
import torch.nn as nn
import torch.utils.model_zoo as model_zoo
import math
import torch.nn.functional as F
import util

__all__ = [
    'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',
    'vgg19_bn', 'vgg19','vgg16_mix3'
]

model_urls = {
    'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth',
    'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth',
    'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth',
    'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth',
    'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth',
    'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth',
    'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth',
    'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth',
}

class BinActive(torch.autograd.Function):
    '''
    Binarize the input activations and calculate the mean across channel dimension.
    '''
    def forward(self, input):
        self.save_for_backward(input)
        size = input.size()
        input = input.sign()
        return input

    def backward(self, grad_output):
        input, = self.saved_tensors
        grad_input = grad_output.clone()
        grad_input[input.ge(1)] = 0
        grad_input[input.le(-1)] = 0
        return grad_input

class BinConv2d(nn.Module):
    def __init__(self, input_channels, output_channels,
            kernel_size=-1, stride=-1, padding=-1, groups=1, dropout=0,
            Linear=False):
        super(BinConv2d, self).__init__()
        self.layer_type = 'BinConv2d'
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        self.dropout_ratio = dropout

        if dropout!=0:
            self.dropout = nn.Dropout(dropout)
        self.Linear = Linear
        if not self.Linear:
            self.bn = nn.BatchNorm2d(input_channels, eps=1e-4, momentum=0.1, affine=True)
            self.conv = nn.Conv2d(input_channels, output_channels,
                    kernel_size=kernel_size, stride=stride, padding=padding, groups=groups)
        else:
            self.bn = nn.BatchNorm1d(input_channels, eps=1e-4, momentum=0.1, affine=True)
            self.linear = nn.Linear(input_channels, output_channels)
        self.relu = nn.LeakyReLU(inplace=True)

    def forward(self, x):
        x = self.bn(x)
        x = BinActive()(x)
        if self.dropout_ratio!=0:
            x = self.dropout(x)
        if not self.Linear:
            x = self.conv(x)
        else:
            x = self.linear(x)
        x = self.relu(x)
        return x

class VGG(nn.Module):

    def __init__(self, features, num_classes=1470, image_size=224):
        super(VGG, self).__init__()
        self.features = features
        self.image_size = image_size

        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, num_classes)
        )
        self._initialize_weights()

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        x = F.sigmoid(x) #归一化到0-1
        x = x.view(-1,7,7,30)
        return x

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()

def make_layers(cfg, batch_norm=False):
    layers = []
    bin_range = [0,0]
    pointer = 0
    in_channels = 3
    s = 1
    first_flag=True
    first_bin_flag = True
    for v in cfg:
        s=1
        if (v==64 and first_flag):
            s=1
            first_flag=False
        if str(v)[-1] == 'B':
            cout = int(v[:-1])
            layers += [BinConv2d(in_channels,cout,kernel_size=3, stride=s, padding=1, groups=1)]
            in_channels = cout
            if first_bin_flag:
                bin_range[0] = pointer
                first_bin_flag = False
            pointer += 1
        elif v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, stride=s, padding=1,groups=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
            pointer += 1
    bin_range[1] = pointer-1
    return nn.Sequential(*layers),bin_range

def conv_bn_relu(in_channels,out_channels,kernel_size=3,stride=2,padding=1):
    return nn.Sequential(
        nn.Conv2d(in_channels,out_channels,kernel_size=kernel_size,padding=padding,stride=stride),
        nn.BatchNorm2d(out_channels),
        nn.ReLU(True)
    )

cfg = {
    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
    'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
    'F': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', '512B', '512B', 512, 'M']
   # 'F': [64, '64B', 'M', '128B', '128B', 'M', '256B', '256B', '256B', 'M', '512B', '512B', '512B', 'M', '512B', '512B', 512, 'M']
}

def vgg11(pretrained=False, **kwargs):
    """VGG 11-layer model (configuration "A")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['A']), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg11']))
    return model

def vgg11_bn(pretrained=False, **kwargs):
    """VGG 11-layer model (configuration "A") with batch normalization

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['A'], batch_norm=True), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg11_bn']))
    return model

def vgg13(pretrained=False, **kwargs):
    """VGG 13-layer model (configuration "B")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['B']), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg13']))
    return model

def vgg13_bn(pretrained=False, **kwargs):
    """VGG 13-layer model (configuration "B") with batch normalization

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['B'], batch_norm=True), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg13_bn']))
    return model

def vgg16(pretrained=False, **kwargs):
    """VGG 16-layer model (configuration "D")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['D']), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg16']))
    return model

def vgg16_bn(pretrained=False, **kwargs):
    """VGG 16-layer model (configuration "D") with batch normalization

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['D'], batch_norm=True), **kwargs)
    if pretrained:
        model.load_state_dict(torch.load('vgg16_bn-6c64b313.pth'))
    return model

def vgg19(pretrained=False, **kwargs):
    """VGG 19-layer model (configuration "E")

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['E']), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg19']))
    return model

def vgg19_bn(pretrained=False, **kwargs):
    """VGG 19-layer model (configuration 'E') with batch normalization

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = VGG(make_layers(cfg['E'], batch_norm=True), **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg19_bn']))
    return model

def vgg16_mix3(pretrained=False, **kwargs):
    """VGG 16-lyaer model, last 3 layer binarized

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    features,bin_range = make_layers(cfg['F'], batch_norm=False)
    model = VGG(features, **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['vgg16']))
    return model,bin_range

def test():
    import torch
    from torch.autograd import Variable
    model,bin_range = vgg16_mix3()
    model.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 1470),
        )
    print(model.classifier[6])
    print(model)
    print(bin_range)
    bin_op = util.BinOp(model,bin_range)
    img = torch.rand(2,3,224,224)
    img = Variable(img)
    output = model(img)
    print(output.size())

if __name__ == '__main__':
    test()


class Square:
    """each square has 3 states: empty, white, black."""
    x: int = None
    y: int = None
    EMPTY = 0
    WHITE = 1
    BLACK = 2
    state = EMPTY

    def __init__(self, x=x, y=y, state=EMPTY):
        self.x = x
        self.y = y
        self.state = state

    def __repr__(self):
        return f"{self.x}{self.y}{self.state}"

    def __eq__(self, other):
        if self.state == other:
            return True
        elif type(other) == Square:
            return self.state == other.state

    def __hash__(self):
        return hash(self.state)

    def flip(self):
        if self.state == Square.WHITE:
            self.state = Square.BLACK
        elif self.state == Square.BLACK:
            self.state = Square.WHITE

# shorthands
WHITE = Square.WHITE
BLACK = Square.BLACK
EMPTY = Square.EMPTY

class Position:
    """Efficient 9x9 board representation. """
    board = None
    move_stack = []
    turn = WHITE
    def __init__(self, board=None):
        if board is None:
            self.board = [[Square(__, _, EMPTY) for _ in range(9)] for __ in range(9)]
        else:
            self.board = board
        self.move_stack = []

    def __str__(self):
        return UTI(self).fen()

    def check_ttt_won(self, board):
        """board is 3x3; check if won and color of winner."""
        for i in range(3):
            if board[i][0] != EMPTY and board[i][0] == board[i][1] == board[i][2]:
                return True, board[i][0]
            if board[0][i] != EMPTY and board[0][i] == board[1][i] == board[2][i]:
                return True, board[0][i]
        if board[0][0] != EMPTY and board[0][0] == board[1][1] == board[2][2]:
            return True, board[0][0]
        if board[0][2] != EMPTY and board[0][2] == board[1][1] == board[2][0]:
            return True, board[0][2]
        return False, None

    def in_sector(self, sq: Square):
        """Check what sector the move is in. """
        return sq.x % 3, sq.y % 3

    def is_sector_equal(self, x, y):
        return x.x == y.x and x.y == y.y

    def sector_to_global(self, x, y, a, b):
        """Convert sector coordinates to global coordinates.
        x, y -> sector coordinates, a, b -> local coordinates"""
        return x * 3 + a, y * 3 + b

    def get_sector_squares(self, x, y):
        TOP_LEFT_x = 3 * x; TOP_LEFT_y = 3 * y
        BOTTOM_RIGHT_x = 3 * x + 2; BOTTOM_RIGHT_y = 3 * y + 2
        board = [[self.board[i][j] for j in range(TOP_LEFT_y, BOTTOM_RIGHT_y + 1)]
                 for i in range(TOP_LEFT_x, BOTTOM_RIGHT_x + 1)]
        return board

    def check_sector_playable(self, x, y):
        """check if the sector is still playable
        a sector is a 3x3 square.
        x: 0-2, y: 0-2
        """
        board = self.get_sector_squares(x, y)
        won = self.check_ttt_won(board)
        if won[0]:
            return False, won[1]
        for i in board:
            for j in i:
                if j == EMPTY:
                    return True, None
        return False, None

    def last_move(self):
        """Return the last move. """
        if len(self.move_stack) == 0:
            return None
        return self.move_stack[-1]

    def make_move(self, move):
        """Make a move. """
        self.move_stack.append(move)
        if move.state is None or move.state == EMPTY:
            move.state = self.turn
        self.board[move.x][move.y] = move.state
        self.turn = BLACK if self.turn == WHITE else WHITE
        # check for sector captured
        for i in range(3):
            for j in range(3):
                info = self.check_sector_playable(i, j)
                if not info[0] and info[1] is not None:
                    # sector captured
                    # convert sector coordinates to global coordinates
                    for a in range(3):
                        for b in range(3):
                            self.board[self.sector_to_global(i, j, a, b)] = info[1]

    def undo_move(self):
        """Undo the last move. """
        move = self.move_stack.pop()
        self.board[move.x][move.y] = EMPTY
        self.turn = BLACK if self.turn == WHITE else WHITE

    def is_game_over(self):
        """Check if the game is over. """
        last_move = self.last_move()
        if last_move is None:
            return False, None
        # Construct large 3x3 board made of sectors
        sectors = [[EMPTY for _ in range(3)] for _ in range(3)]
        for i in range(3):
            for j in range(3):
                info = self.check_sector_playable(i, j)
                if not info[0] and info[1] is not None:
                    sectors[i][j] = info[1]
                else:
                    sectors[i][j] = EMPTY
        won = self.check_ttt_won(sectors)
        if won[0]:
            return True, won[1]
        return False, None

class MoveGen:
    """Generate all possible moves for a given position. """
    pos = Position()
    def __init__(self, position=Position()):
        self.pos = position

    def legal_moves(self):
        """Generate all possible moves for a given position. """
        pos = self.pos
        turn = pos.turn
        last_move = pos.last_move()
        if last_move is None:
            return [Square(i, j, WHITE) for i in range(9) for j in range(9)]
        ALL_SQUARES_OK = False
        sector = pos.in_sector(last_move)
        if not pos.check_sector_playable(sector[0], sector[1]):
            ALL_SQUARES_OK = True
        moves = []
        if ALL_SQUARES_OK:
            for i in range(9):
                for j in range(9):
                    if pos.board[i][j] == EMPTY:
                            moves.append(Square(i, j, turn))
        else:
            # Only this sector is playable
            board = pos.get_sector_squares(sector[0], sector[1])
            for i in range(3):
                for j in range(3):
                    if board[i][j] == EMPTY:
                        x, y = pos.sector_to_global(sector[0], sector[1], i, j)
                        moves.append(Square(x, y, turn))
        return moves

class UTI:
    """Universal TTT Interface. """
    pos = Position()
    def __init__(self, position=Position()):
        self.pos = position

    def fen(self):
        """Output the current position. """
        pos = self.pos
        fen = ""
        for i in range(9):
            for j in range(9):
                fen += str(pos.board[i][j])
                fen += "/"
        fen += f" {pos.turn}"
        return fen

    def from_fen(self, fen):
        """Load a position from a fen string. """
        fen = fen.split()
        board = fen[0].split("/")
        turn = fen[1]
        for i in range(9):
            for j in range(9):
                if board[i][j][-1] == "0":
                    board[i][j] = EMPTY
                elif board[i][j][-1] == "1":
                    board[i][j] = WHITE
                elif board[i][j][-1] == "2":
                    board[i][j] = BLACK
        pos = Position(board)
        pos.turn = turn
        self.pos = pos
        return pos


"""
示例8-16 没有指向对象的引用时，监视对象生命结束时的情形
"""

import weakref

s1 = {1, 2, 3}
s2 = s1

def bye():
    print("Gone with the wind")

ender = weakref.finalize(s1, bye) # 在s1 引用的对象上注册bye回调; finalize持有集合{1, 2, 3}的弱引用
print(ender.alive) # True ;  调用finalize 对象之前， .alive 属性的值为True

del  s1
print(ender.alive) # True # del不删除对象，而是删除对象的引用

s2 = "spam" # 重新绑定最后一个引用s2， 让集合{1, 2, 3}无法获取；对象被销毁了，调用了bye回调
print(ender.alive) # False; ender.alive属性变为了False


#!/usr/bin/python

from elasticsearch import Elasticsearch
import subprocess
from xlrd import open_workbook

def readXlsx(fileName):
	book = open_workbook(fileName)
	sheet = book.sheet_by_index(1)

	# read header values into the list
	keys = [sheet.cell(0, col_index).value for col_index in xrange(sheet.ncols)]

	dict_list = []
	for row_index in xrange(1, sheet.nrows):
		d = {keys[col_index]: sheet.cell(row_index, col_index).value for col_index in xrange(sheet.ncols)}
		dict_list.append(d)

	#for row in dict_list:
	#	print row['Action']
	return dict_list

def searchElasticserach(indices, query, startTime, endTime):
	# Search the elasticsearch db.i
	es = Elasticsearch(timeout=600)
	res = es.search(index=indices,body={
	  "sort": [
	    {
	      "@timestamp": {
	        "order": "desc",
	        "unmapped_type": "boolean"
	      }
	    }
	  ],
	  "highlight": {
	    "pre_tags": [
	      "@kibana-highlighted-field@"
	    ],
	    "post_tags": [
	      "@/kibana-highlighted-field@"
	    ],
	    "fields": {
	      "*": {}
	    },
	    "fragment_size": 2147483647
	  },
	  "query": {
	    "filtered": {
	      "query": {
		"query_string": {
		  "query": query
		}
	      },
	      "filter": {
		"bool": {
		  "must": [
		    {
		      "range": {
			"@timestamp": {
			  "gte": startTime,
			  "lte": endTime
			}
		      }
		    }
		  ],
		  "must_not": []
		}
	      }
	    }
	  },
	  "size": 1000000,
	  "fields": [
	    "*",
	    "_source"
	  ],
	  "script_fields": {},
	  "fielddata_fields": [
	    "@timestamp"
	  ]
	}
	)
	return res['hits']['hits']

# This method adds or modifies the dictionary supplied
def modify(data,dictName):
	if isinstance(dictName, dict):
		if not data in dictName:
			dictName[data] = {'count':1}
		else:
			dictName[data]['count'] +=1
	return dictName

def createErrorDict(dictName,value):
	test[dictName] = value
	errorDict.update(test)

def getRegEx(dict_list):
	regEx = []
	for line in dict_list:
                regEx.append(line['Reg-Ex'])
	return regEx

def resultsPerHosts(res, hosts, dict_list):
	host = {}
	regEx = getRegEx(dict_list)
	for hit in res:
		host = modify(hit['_source']['host'],hosts)
		for i in  regEx:
			if i in hit['_source']['message']:
				#print "inside if"
				tmp = hit['_source']['host']
				host = modify(hit['_source']['message'],hosts[tmp])
	return host

def modifySeverity(res,severity,changeSeverity):
	for hit in res:
	       if hit['_source']['ar_tag'] in changeSeverity:
	               severity[changeSeverity[hit['_source']['ar_tag']]] += 1
	       else:
	               severity[hit['_source']['ar_severity']] += 1
	return severity


# -*- coding: gb18030 -*-
#
# $Id: Spell_Item_Change_Body.py,v 1.2 2008-08-14 02:32:06 zhangyuxing Exp $

"""
在规定位置使用物品
"""

from SpellBase import *
import cschannel_msgs
import ShareTexts as ST
from Spell_BuffNormal import Spell_ItemBuffNormal
import csstatus
import BigWorld
import csdefine
import csconst
from Spell_Item import Spell_Item

# 根据角色的性别、职业，从物品的param参数中得到适当的模型组合ID
param_map = {
	0:{
	16:"param1",		# 战士
	32:"param2",		# 剑客
	48:"param3",		# 射手
	64:"param4",		# 法师
	},
	1:{
	16:"param5",
	32:"param6",
	48:"param7",
	64:"param8",
	},
}

class Spell_ItemChangeHeadAbout( Spell_Item ):
	"""
	"""
	def __init__( self ):
		"""
		构造函数。
		"""
		Spell_Item.__init__( self )

	def init( self, dict ):
		"""
		读取技能配置
		@param dict: 配置数据
		@type  dict: python dict
		"""
		Spell_Item.init( self, dict )

	def receive( self, caster, receiver ):
		"""
		virtual method.
		法术到达所要做的事情
		"""
		# self.drawChangeBody( caster, receiver )
		uid = caster.queryTemp( "item_using" )
		item = caster.getItemByUid_( uid )
		if item:
			param = param_map[receiver.getGender()][receiver.getClass()]
			modnum = int(item.query( param ))
			receiver.setTemp( "headAboutModNum", modnum )
		Spell_Item.receive( self, caster, caster )


import os
from pathlib import Path

import pandas as pd

# In the challenge, you are predicting item sales at stores in various locations for two 28-day time periods.
#
# Target: 'sales' column
#
# Time period: 2011-01-29 to 2016-06-19
#
# 3 data sources:
# - data_calendar: date related features
#     - Daily resolution
#     - Keys: date, wm_yr_wk
# - data_sales_train: historical daily unit sales per product and store
#     - Daily resolution
#     - Keys: store_id, item_id
# - data_sell_prices: Price of the products sold per store and date
#     - Weekly averaged resolution
#     - Keys: store_id, item_id, wm_yr_wk

def get_data(keep_day_ratio, n_item_ids):
    # Download data from: https://www.kaggle.com/c/m5-forecasting-accuracy/data and store under the 'data' directory

    data_dir = Path(os.path.join(os.getcwd(), 'data', 'm5-forecasting-accuracy'))

    ####
    # Load 'calendar.csv'
    file_calendar = os.path.join(data_dir, 'calendar.csv')
    data_calendar = pd.read_csv(file_calendar)
    data_calendar['date'] = pd.to_datetime(data_calendar['date'])

    ####
    # Load 'sales_train_validation.csv'
    # Contains the historical daily unit sales data per product and store [d_1 - d_1913]
    # row_id | item_id | department_id | category_id | store_id | state_id | unit_sale_data d_1 - d_1913
    # Contains the historical daily unit sales data per product and store
    file_sales_train_validation = os.path.join(data_dir, 'sales_train_validation.csv')
    data_sales = pd.read_csv(file_sales_train_validation)

    ####
    # Load 'sell_prices.csv'
    #Contains information about the price of the products sold per store and date.
    file_sell_prices = os.path.join(data_dir, 'sell_prices.csv')
    data_sell_prices = pd.read_csv(file_sell_prices)

    # filter out days in order to reduce dataset so it fits into a single laptops RAM
    if keep_day_ratio is not None:
        non_day_columns = [col for col in data_sales.columns if 'd_' not in col]
        day_columns = [col for col in data_sales.columns if 'd_' in col]
        day_columns = day_columns[:int(len(day_columns) * keep_day_ratio)]
        data_sales = data_sales[non_day_columns + day_columns]

    # filter out items in order to reduce dataset so it fits into a single laptops RAM
    if n_item_ids is not None:
        keep_item_ids = data_sales['item_id'].unique()[:n_item_ids]
        data_sales = data_sales[data_sales['item_id'].isin(keep_item_ids)]

    # Transform sales data from wide in a time-series like long format via 'pd.melt'
    keep_cols = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
    data_sales_ts = pd.melt(data_sales,
                            id_vars=keep_cols,
                            var_name='day',
                            value_name='sales'
                            )

    # Convert the integer 'day' column into a proper datetime type 'date' column
    data_sales_ts['day_since_start_date'] = data_sales_ts['day'].apply(lambda x: int(x[2:])) - 1
    data_sales_ts['day_since_start_date_timedelta'] = pd.to_timedelta(data_sales_ts['day_since_start_date'], unit='d')
    data_sales_ts['date'] = pd.Timestamp("2011-01-29") + data_sales_ts['day_since_start_date_timedelta']
    data_sales_ts = data_sales_ts.drop(columns=['day', 'day_since_start_date', 'day_since_start_date_timedelta'])

    # merge sales data and date features
    data_full = data_sales_ts.merge(data_calendar.drop(columns=['d']), how='left', on=['date'])

    # merge data_full and sell_prices per week
    data_full = data_full.merge(data_sell_prices, how='left', on=['item_id', 'store_id', 'wm_yr_wk'])
    data_full = data_full.rename(columns={'sell_price': 'sell_price_per_week'})

    return data_full