import bpy
import gpu
import math
import time
import mathutils
from bgl import *
from mathutils import Vector
from math import cos, sin, pi, radians, degrees
from bpy_extras import view3d_utils
from gpu_extras.batch import batch_for_shader
from bpy.props import IntProperty, FloatProperty

from ..graphics.draw import render_text, render_quad, draw_border_lines
from ..utils.geo import get_blf_text_dims

from ...preferences import get_preferences
from ...addon.utility import method_handler
from ...utils.blender_ui import get_dpi, get_dpi_factor

handle = None

class Data():
    '''Global communications from operator to ui modal.'''
    # Fade Variables
    start_time = 0
    time_difference = 0
    hold_complete = False

    # Transitions
    active_count = 0

    # Display Data
    draw_data = []

    draw_bg = False
    draw_border = False

    # Trigger a global reset on the running modal
    reset_all = False

class Master():
    '''Used from the modal to launch the drawing.'''

    def receive_draw_data(self, draw_data=[]):

        Data.draw_data = draw_data

    def draw(self, draw_bg=False, draw_border=False):

        prefs = get_preferences()

        if prefs.ui.Hops_operator_display:

            Data.draw_bg = draw_bg
            Data.draw_border = draw_border

            # Reuse the same modal check
            if Data.active_count == 0:
                Data.active_count += 1
                Data.reset_all = False
                bpy.ops.hops.ui_draw('INVOKE_DEFAULT')
            else:
                Data.reset_all = True

class HOPS_OT_UI_Draw(bpy.types.Operator):
    '''UI Modal'''

    bl_idname = "hops.ui_draw"
    bl_label = "Drawing for operators"
    bl_options = {"INTERNAL"}

    def __init__(self):

        self.shader = None
        self.timer = None
        self.prefs = get_preferences()
        self.first_draw = False

    def invoke(self, context, event):

        # Globals
        Data.start_time = time.time()
        Data.hold_complete = False
        Data.time_difference = 0

        # Fail Safe
        self.start_time = time.time()
        self.max_time = self.prefs.ui.Hops_operator_display_time + self.prefs.ui.Hops_operator_display_time + 3

        #UI System
        self.shader = Shader(context)
        self.timer = context.window_manager.event_timer_add(0.025, window=context.window)

        context.window_manager.modal_handler_add(self)

        return {'RUNNING_MODAL'}

    def modal(self, context, event):

        # Reuse this modal if a new draw data is invoked
        if Data.reset_all == True:
            Data.reset_all = False
            Data.start_time = time.time()
            Data.hold_complete = False
            Data.time_difference = 0
            self.start_time = time.time()
            self.max_time = self.prefs.ui.Hops_operator_display_time + self.prefs.ui.Hops_operator_display_time + 3
            self.first_draw = False

        # Fail Safe
        if time.time() - self.start_time > self.max_time:
            self.finished(context)
            return {'FINISHED'}

        # Get the time difference
        Data.time_difference = time.time() - Data.start_time

        # Remove modal if another starts
        if Data.active_count > 1:
            self.finished(context)
            return {'FINISHED'}

        # Hold the UI on screen until display time is completed
        if Data.hold_complete == False:

            if Data.time_difference >= self.prefs.ui.Hops_operator_display_time:
                Data.hold_complete = True
                Data.time_difference = 0
                Data.start_time = time.time()

            elif Data.time_difference <= self.prefs.ui.Hops_operator_display_time:

                # Redraw the viewport
                if context.area != None:
                    if self.first_draw == False:
                        self.first_draw = True
                        context.area.tag_redraw()
                return {'PASS_THROUGH'}

        # Fade is finished
        if Data.time_difference > self.prefs.ui.Hops_operator_fade:
            self.finished(context)
            return {'FINISHED'}

        # Redraw the viewport
        if context.area != None:
            context.area.tag_redraw()

        return {'PASS_THROUGH'}

    def finished(self, context):
        '''Remove the timer, shader, and reset Data'''

        Data.time_difference = 0
        Data.active_count -= 1

        if self.timer != None:
            context.window_manager.event_timer_remove(self.timer)

        if self.shader != None:
            self.shader.destroy()
            del self.shader

        if context.area != None:
            context.area.tag_redraw()

class Shader():

    def __init__(self, context):

        self.context = context
        self.banner = Banner(context=self.context)
        self.setup_handle()

    def setup_handle(self):
        '''Setup the draw handle for the UI'''

        global handle
        handle = bpy.types.SpaceView3D.draw_handler_add(self.safe_draw, (self.context, ), "WINDOW", "POST_PIXEL")

    def safe_draw(self, context):
        method_handler(self.draw,
            arguments = (context,),
            identifier = 'UI Framework',
            exit_method = self.destroy)

    def draw(self, context):

        self.banner.draw()

    def destroy(self):

        global handle

        if handle != None:
            bpy.types.SpaceView3D.draw_handler_remove(handle, "WINDOW")
            handle = None
            if self.banner:
                del self.banner
            return True

        else:
            return False

class Banner():
    '''Drawing class for operator UI.'''

    def __init__(self, context):

        self.context = context
        self.prefs = get_preferences()
        self.scale_factor = get_dpi_factor()
        self.setup_colors()

        # Font sizes
        self.header_font_size = 24
        self.body_font_size = 14

        # Font dims
        self.header_font_dims = get_blf_text_dims(text="Sample", size=self.header_font_size)
        self.body_font_dims = get_blf_text_dims(text="Sample", size=self.body_font_size)

        # Paddings
        self.text_padding = 6 * self.scale_factor
        self.column_gap = 150 * self.scale_factor
        self.side_padding = 15 * self.scale_factor
        self.padding_to_bottom_line = 10 * self.scale_factor
        self.line_width = 2

        # Offsets
        self.prefs_offset = (
            self.prefs.ui.Hops_operator_ui_offset[0] * self.scale_factor,
            self.prefs.ui.Hops_operator_ui_offset[1] * self.scale_factor)

        self.height_to_bottom = self.prefs_offset[1]

        # Totals
        self.total_width = self.get_total_width()
        self.total_height =self.get_total_height()
        self.left_offset = (self.context.area.width * .5) - (self.total_width * .5) + self.prefs_offset[0]

    #######################################
    # SETUPS
    #######################################

    def get_total_width(self):
        '''Get the total wdth of the box.'''

        # Get the widest text width
        if Data.draw_data != []:

            # Only containts header
            if len(Data.draw_data) == 1:
                text = Data.draw_data[0][0]
                return get_blf_text_dims(text=text, size=self.header_font_size)[0] + self.side_padding * 2

            # Header plus other data
            else:
                longest = ""
                for i in range(len(Data.draw_data)):

                    # Only one item in list
                    if len(Data.draw_data[i]) == 1:
                        if len(Data.draw_data[i][0]) > len(longest):
                            longest = Data.draw_data[i][0]

                    # Two items in list
                    elif len(Data.draw_data[i]) == 2:
                        compare = str(Data.draw_data[i][0]) + str(Data.draw_data[i][1])
                        if len(compare) > len(longest):
                            longest = compare

                return get_blf_text_dims(text=str(longest), size=self.body_font_size)[0] + self.column_gap

    def get_total_height(self):
        '''Get the total height of the box.'''

        # Single line label only
        if len(Data.draw_data) == 1:
            return self.header_font_dims[1] + self.text_padding * 3

        # Full label
        else:

            # For each text add padding for top
            txt_padding_total = self.text_padding * (len(Data.draw_data) - 2)

            # Both divisor lines
            divisor_total = self.line_width * 2
            divisor_total += self.text_padding * 2
            divisor_total += self.padding_to_bottom_line * 2

            # All the text heights
            text_height_total = self.header_font_dims[1]
            text_height_total += self.body_font_dims[1] * (len(Data.draw_data) - 1)

            return text_height_total + txt_padding_total + divisor_total

    #######################################
    # COLORS
    #######################################

    def setup_colors(self):

        # Original
        self.prefs_colors = []
        self.prefs_colors.append(self.prefs.color.Hops_UI_text_color)
        self.prefs_colors.append(self.prefs.color.Hops_UI_secondary_text_color)
        self.prefs_colors.append(self.prefs.color.Hops_UI_text_highlight_color)
        self.prefs_colors.append(self.prefs.color.Hops_UI_cell_background_color)
        self.prefs_colors.append(self.prefs.color.Hops_UI_border_color)

    def set_colors(self):

        self.Hops_UI_text_color            = Vector((self.prefs_colors[0][0], self.prefs_colors[0][1], self.prefs_colors[0][2], self.prefs_colors[0][3]))
        self.Hops_UI_secondary_text_color  = Vector((self.prefs_colors[1][0], self.prefs_colors[1][1], self.prefs_colors[1][2], self.prefs_colors[1][3]))
        self.Hops_UI_text_highlight_color  = Vector((self.prefs_colors[2][0], self.prefs_colors[2][1], self.prefs_colors[2][2], self.prefs_colors[2][3]))
        self.Hops_UI_cell_background_color = Vector((self.prefs_colors[3][0], self.prefs_colors[3][1], self.prefs_colors[3][2], self.prefs_colors[3][3]))
        self.Hops_UI_border_color          = Vector((self.prefs_colors[4][0], self.prefs_colors[4][1], self.prefs_colors[4][2], self.prefs_colors[4][3]))

    def handle_fade(self):

        if Data.hold_complete == True:
            if self.prefs.ui.Hops_operator_fade > 0:

                original_alpha = self.prefs_colors[0][3]
                alpha = self.Hops_UI_text_color[3] - ((original_alpha / self.prefs.ui.Hops_operator_fade) * Data.time_difference)
                self.Hops_UI_text_color = Vector((self.prefs_colors[0][0], self.prefs_colors[0][1], self.prefs_colors[0][2], alpha))

                original_alpha = self.prefs_colors[1][3]
                alpha = self.Hops_UI_secondary_text_color[3] - ((original_alpha / self.prefs.ui.Hops_operator_fade) * Data.time_difference)
                self.Hops_UI_secondary_text_color = Vector((self.prefs_colors[1][0], self.prefs_colors[1][1], self.prefs_colors[1][2], alpha))

                original_alpha = self.prefs_colors[2][3]
                alpha = self.Hops_UI_text_highlight_color[3] - ((original_alpha / self.prefs.ui.Hops_operator_fade) * Data.time_difference)
                self.Hops_UI_text_highlight_color  = Vector((self.prefs_colors[2][0], self.prefs_colors[2][1], self.prefs_colors[2][2], alpha))

                original_alpha = self.prefs_colors[3][3]
                alpha = self.Hops_UI_cell_background_color[3] - ((original_alpha / self.prefs.ui.Hops_operator_fade) * Data.time_difference)
                self.Hops_UI_cell_background_color = Vector((self.prefs_colors[3][0], self.prefs_colors[3][1], self.prefs_colors[3][2], alpha))

                original_alpha = self.prefs_colors[4][3]
                alpha = self.Hops_UI_border_color[3] - ((original_alpha / self.prefs.ui.Hops_operator_fade) * Data.time_difference)
                self.Hops_UI_border_color = Vector((self.prefs_colors[4][0], self.prefs_colors[4][1], self.prefs_colors[4][2], alpha))

    #######################################
    # DRAW
    #######################################

    def draw(self):

        # Resets
        self.height_to_bottom = self.prefs_offset[1]
        self.total_width = self.get_total_width()
        self.total_height =self.get_total_height()
        self.left_offset = (self.context.area.width * .5) - (self.total_width * .5) + self.prefs_offset[0]

        if Data.draw_data != []:

            # Colors
            self.set_colors()
            self.handle_fade()

            # Draw header and lines
            if len(Data.draw_data) > 1:
                self.draw_background()
                self.draw_divisor_line()
                self.draw_body_text()
                self.draw_divisor_line()
                self.draw_header_text()

            # Draw only header
            else:
                self.draw_background()
                self.draw_header_text()

    def draw_background(self):

        # Setup for background and border : TL, BL, TR, BR
        quad = (
            (self.left_offset                   , self.height_to_bottom + self.total_height),
            (self.left_offset                   , self.height_to_bottom),
            (self.left_offset + self.total_width, self.height_to_bottom + self.total_height),
            (self.left_offset + self.total_width, self.height_to_bottom))

        # Draw background
        if Data.draw_bg:
            render_quad(quad=quad,
                        color=self.Hops_UI_border_color,
                        bevel_corners=True)

        # Draw border
        if Data.draw_border:
            draw_border_lines(vertices=quad, width=self.prefs.ui.Hops_operator_border_size, color=self.Hops_UI_cell_background_color)

    def draw_divisor_line(self):

        left_point  = (self.left_offset + self.side_padding                   , self.height_to_bottom + self.padding_to_bottom_line)
        right_point = (self.left_offset - self.side_padding + self.total_width, self.height_to_bottom + self.padding_to_bottom_line)

        self.height_to_bottom += self.line_width + self.text_padding + self.padding_to_bottom_line

        draw_divider_lines(
            vertices=[left_point, right_point],
            width=self.line_width,
            color=self.Hops_UI_cell_background_color,
            bevel_corners=False,
            format_lines=False)

    def draw_body_text(self):

        # Text items for the body
        for index, item in enumerate(Data.draw_data[1:]):
            # Left or Right
            count = 0
            for string in item:

                if type(string) == float:
                    string = "{0:.3f}".format(string)
                string = str(string)

                # Left Side
                if count == 0:
                    pos = (self.left_offset + self.side_padding, self.height_to_bottom)
                    render_text(
                        text=string,
                        position=pos,
                        size=self.body_font_size,
                        color=self.Hops_UI_text_color)

                # Right Side
                elif count == 1:
                    text_dims = get_blf_text_dims(text=string, size=self.body_font_size)
                    pos = (self.left_offset + self.total_width - text_dims[0] - self.side_padding, self.height_to_bottom)
                    render_text(
                        text=string,
                        position=pos,
                        size=self.body_font_size,
                        color=self.Hops_UI_secondary_text_color)

                count += 1

            self.height_to_bottom += self.body_font_dims[1] + self.text_padding
        self.height_to_bottom -= self.text_padding * 2

    def draw_header_text(self):

        string = Data.draw_data[0][0]
        pos = (self.left_offset + self.side_padding, self.height_to_bottom)

        # Single line label only
        if len(Data.draw_data) == 1:
            pos = (self.left_offset + self.side_padding, self.height_to_bottom + self.text_padding * 2)

        render_text(
            text=string,
            position=pos,
            size=self.header_font_size,
            color=self.Hops_UI_secondary_text_color)

def draw_divider_lines(vertices=[], width=1, color=(0,0,0,1), bevel_corners=True, format_lines=False):
    '''Render quads passed in. \n
        Top Left, Bottom Left, Top Right, Bottom Right
    '''

    shader = gpu.shader.from_builtin('2D_UNIFORM_COLOR')
    glEnable(GL_LINE_SMOOTH)
    glEnable(GL_BLEND)
    glLineWidth(width)
    batch = batch_for_shader(shader, 'LINE_STRIP', {"pos": vertices})
    shader.bind()
    shader.uniform_float("color", color)
    batch.draw(shader)

    glLineWidth(width)
    glDisable(GL_BLEND)

    del shader
    del batch

from bpy.app.handlers import persistent
@persistent
def remove_op_draw_shader(dummy):
    global handle
    if handle != None:
        bpy.types.SpaceView3D.draw_handler_remove(handle, "WINDOW")
        handle = None

    # Reset draw data
    Data.start_time = 0
    Data.time_difference = 0
    Data.hold_complete = False
    Data.active_count = 0
    Data.draw_data = []
    Data.draw_bg = False
    Data.draw_border = False
    Data.reset_all = False


import os
import random
import uuid
import time
import numpy as np
from datetime import datetime
import math
# video edit
from moviepy.editor import ImageSequenceClip, TextClip, VideoFileClip,\
    concatenate_videoclips#, ImageClip, CompositeVideoClip
from moviepy.audio.AudioClip import AudioArrayClip
#face track
import cv2
#voice
import librosa
from AudioLib.AudioProcessing import AudioProcessing

#get facial classifiers
CCPATH = './Lib/site-packages/cv2/haar-cascade-files-master/'
face_cascade = cv2.CascadeClassifier(CCPATH +'haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier(CCPATH +'haarcascade_eye.xml')

class VideoEditor:

    DEFAULT_MASK_PATH = os.path.abspath("./masks/1.png")
    TEMP_FOLDER_PATH = os.path.abspath("./temp_files")
    DEFAULT_FILENAME = "output.mp4"
    BLURE_FULL_IMG = (120, 120)
    BLURE_OLD_FACES = (75, 75)
    CURRENT_DIR = None

    def __init__(self, input_video, mask_path = None, output_video = None):

        # current dir reset
        if VideoEditor.CURRENT_DIR is None:
            VideoEditor.CURRENT_DIR = os.getcwd()
        else:
            os.chdir(VideoEditor.CURRENT_DIR)

        # check if input video exist
        if not os.path.isfile(input_video):
            raise Exception('Error : input video path not found')
        self.input_video = os.path.abspath(input_video)

        # make tmp folder
        if not os.path.isdir(VideoEditor.TEMP_FOLDER_PATH):
            os.mkdir(VideoEditor.TEMP_FOLDER_PATH)
        now = datetime.now()
        self.tmp_folder = os.path.abspath(os.path.join(VideoEditor.TEMP_FOLDER_PATH,\
            now.strftime("%Y_%m_%d_%H_%M_%S___") + str(uuid.uuid4()) ))
        print("temp folder '{}'".format(self.tmp_folder))
        os.mkdir(self.tmp_folder)

        # output video path
        if output_video is None:
            filename = VideoEditor.DEFAULT_FILENAME
            output_video = os.path.join(self.tmp_folder, filename)
        else:
            output_video = os.path.abspath(output_video)
        self.output_video = output_video

        # mask
        if mask_path is None:
            mask_path = VideoEditor.DEFAULT_MASK_PATH
        print("-Reading mask ...")
        if not os.path.isfile(mask_path):
            raise Exception('Error : mask not found')
        self.mask = cv2.imread(mask_path)
        self.original_witch_h, self.original_witch_w,_ = self.mask.shape
        self.mask_gray = cv2.cvtColor(self.mask, cv2.COLOR_BGR2GRAY)
        _, self.original_mask = cv2.threshold(self.mask_gray, 10, 255, cv2.THRESH_BINARY_INV)
        self.original_mask_inv = cv2.bitwise_not(self.original_mask)

        # default blure area
        self.blure_x_max = self.blure_y_max = 0
        self.blure_x_min = self.blure_y_min = math.inf

        self.hide_full_frame = True
        self.last_faces = None

    def __del__(self):

        print("\n\n")
        # deleting temp folder if empty
        if not any(os.scandir(self.tmp_folder)):
            print('no files found !!')
            print('-deleting {}'.format(self.tmp_folder))
            os.rmdir(self.tmp_folder)

    def hide_face(self):

        # change current directory to temp folder
        os.chdir(self.tmp_folder)

        print("-Reading video ...")
        clip = VideoFileClip(self.input_video)

        print("-Edit video frames ...")
        new_frames = []
        count = 0
        count_process = 0
        clip_frames_count = clip.fps * clip.duration + 1
        proccess_at_fps = 5
        tmp = -1
        start_time = time.monotonic()
        for frame in clip.iter_frames():
            count += 1; tmp += 1

            if tmp == proccess_at_fps:
                tmp = 0

            process = False
            if tmp == 0 or self.last_faces is None:
                count_process += 1; process = True

            frame = self.hide_face_frame(frame, process)
            new_frames.append(frame)

            print("time: {} s - frames percentage: {} %    ".format(\
                round(time.monotonic() - start_time), \
                round(count / clip_frames_count * 100, 2)), end="\r")

        print("processed frames : ( {} / {} )             ".format(count_process,count))
        print("time taken : {} s ".format(round(time.monotonic() - start_time)))
        print("-Generate text ...")
        txt = TextClip("Hey you ! \n\n Join Fadfada \n Today", color='FloralWhite',
        size = clip.size, bg_color='DodgerBlue4',
        font = "Century-Schoolbook-Italic", fontsize= clip.w / 12)

        print("-concatinate ImageSequence ...")
        new_clip = ImageSequenceClip(new_frames, fps=clip.fps)

        # voice
        print("-voice processing ...")
        tasks_count = 4; current_task_id = 1
        print("Loading ( {} / {} )".format(current_task_id, tasks_count), end="\r"); current_task_id += 1
        fs = clip.audio.fps
        audio_arr = clip.audio.copy().to_soundarray()
        # to mono
        print("Loading ( {} / {} )".format(current_task_id, tasks_count), end="\r"); current_task_id += 1
        new_audio_arr = []
        for e in audio_arr:
            new_audio_arr.append((e[0] / 2) + (e[1] / 2))
        audio_arr = np.array(new_audio_arr)
        # effect
        print("Loading ( {} / {} )".format(current_task_id, tasks_count), end="\r"); current_task_id += 1
        y_shifted = librosa.effects.pitch_shift(audio_arr, fs, n_steps=random.randrange(-6, -3))
        sound = AudioProcessing(y_shifted, fs)
        # sound.set_bandpass(300, 3000)
        audio_arr = sound.get_data()
        # to dual
        print("Loading ( {} / {} )".format(current_task_id, tasks_count), end="\r"); current_task_id += 1
        new_audio_arr = []
        for e in audio_arr:
            new_audio_arr.append([e, e])
        audio_arr = np.array(new_audio_arr)
        audio = AudioArrayClip(audio_arr, fps=fs)

        print("-concatinate Video and Audio...")
        new_clip = concatenate_videoclips([new_clip, txt.set_duration(3)]).set_audio(audio)
        new_clip.write_videofile(self.output_video)
        # new_clip.write_videofile(self.output_video, bitrate="3000k")

        # reset current directory
        os.chdir(VideoEditor.CURRENT_DIR)

        return self.output_video

    def hide_face_frame(self, img, detect_faces = False):

        img = img.copy()

        # convert to gray
        img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # find faces in image using classifier
        faces = []
        if detect_faces:
            faces = face_cascade.detectMultiScale(img_gray, 1.3, 5)

        # if there no faces in current frame blur it
        if len(faces) == 0 and self.hide_full_frame:
            img = cv2.blur(img, VideoEditor.BLURE_FULL_IMG)
        else:
            self.hide_full_frame = False

        if self.blure_y_min != math.inf:
            img[self.blure_y_min:self.blure_y_max, self.blure_x_min:self.blure_x_max] =\
                cv2.blur(img[self.blure_y_min:self.blure_y_max, self.blure_x_min:self.blure_x_max],\
                    VideoEditor.BLURE_OLD_FACES)

        # put mask on each face
        for face in faces:
            img = self.put_mask_on_face(img, face)

        # last mask saved
        if len(faces) == 0 and self.last_faces is not None:
            for face in self.last_faces:
                img = self.put_mask_on_face(img, face)
        if len(faces) != 0:
            self.last_faces = faces

        return img

    def put_mask_on_face(self, img, face):

        #get shape of img
        img_h,img_w,_ = img.shape
        X, Y, W, H = face

        #retangle for testing purposes
        #img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)

        #coordinates of face region
        face_w = W
        face_h = H
        face_x1 = X
        face_x2 = face_x1 + face_w
        face_y1 = Y
        face_y2 = face_y1 + face_h

        #witch size in relation to face by scaling
        witch_width = int(1.5 * face_w)
        witch_height = int(witch_width * self.original_witch_h / self.original_witch_w)

        #setting location of coordinates of witch
        # witch_x1 = face_x2 - int(face_w/2) - int(witch_width/2)
        # witch_x2 = witch_x1 + witch_width
        # witch_y1 = face_y1 - int(face_h*1.25)
        # witch_y2 = witch_y1 + witch_height

        witch_x1 = face_x1 - int(face_w*0.11)
        witch_x2 = face_x2 + int(face_w*0.11)
        witch_y1 = face_y1 - int(face_h*0.3) #int(witch_height/6)
        witch_y2 = face_y2 + int(face_h*0.3) #face_y2 + int(witch_height/6)

        #check to see if out of frame

        if witch_x1 < 0:
            witch_x1 = 0
        if witch_y1 < 0:
            witch_y1 = 0
        if witch_x2 > img_w:
            witch_x2 = img_w
        if witch_y2 > img_h:
            witch_y2 = img_h

        mask = self.mask.copy()

        #Account for any out of frame changes
        witch_width = witch_x2 - witch_x1
        witch_height = witch_y2 - witch_y1

        #resize witch to fit on face
        mask = cv2.resize(mask, (witch_width,witch_height), interpolation = cv2.INTER_AREA)
        witch2 = cv2.resize(self.original_mask, (witch_width,witch_height), interpolation = cv2.INTER_AREA)
        mask_inv = cv2.resize(self.original_mask_inv, (witch_width,witch_height), interpolation = cv2.INTER_AREA)

        #take ROI for witch from background that is equal to size of witch image
        roi = img[witch_y1:witch_y2, witch_x1:witch_x2]

        #original image in background (bg) where witch is not present
        roi_bg = cv2.bitwise_and(roi,roi,mask = witch2)
        roi_fg = cv2.bitwise_and(mask,mask,mask=mask_inv)
        dst = cv2.add(roi_bg,roi_fg)

        self.blure_x_min = min(self.blure_x_min, witch_x1)
        self.blure_x_max = max(self.blure_x_max, witch_x2)
        self.blure_y_min = min(self.blure_y_min, witch_y1)
        self.blure_y_max = max(self.blure_y_max, witch_y2)
        #print("{} , {} , {} , {}".format(self.blure_x_min,self.blure_x_max,self.blure_y_min,self.blure_y_max))

        #put back in original image
        img[witch_y1:witch_y2, witch_x1:witch_x2] = dst

        return img

# test section
# all_start_time = time.monotonic()
# x=1
# if x==1:
#     test_video = VideoEditor("./videos/2.mp4","./masks/6.png","./temp_files/2-6.mp4")
#     test_video.hide_face()

#     test_video = VideoEditor("./videos/1.mp4","./masks/6.png","./temp_files/1-6.mp4")
#     test_video.hide_face()
# else:
#     test_video = VideoEditor("./videos/2.mp4","./masks/5.png","./temp_files/last.mp4")
#     img_ = cv2.imread('./imgs/1.jpg')
#     img_ = test_video.hide_face_frame(img_)

#     cv2.imshow('img',img_)
#     cv2.waitKey(0)
#     cv2.destroyAllWindows()
# print("all time taken : {} s".format(round(time.monotonic() - all_start_time)))


import copy

class Paging(object):
    def __init__(self, req, page_num, sum_num, show_num=10, show_page_num=5):
        # 请求路径，做拼接用
        self.path = req.path

        # 用户输入的是否正确，如"aaa"给他定位到第一页
        try:
            self.page_num = int(page_num)
        except:
            self.page_num = 1

        # 数据总数
        self.sum_num = sum_num

        # GET提交的参数,为了保留搜索条件
        # 提交的参数，需要深copy得到一下，因为req.GET不可修改
        try:
            self.param = copy.deepcopy(req.GET)
        # 报错可能是应为是使用flask框架的缘故，所以换个写法。
        except AttributeError:
            self.param = req.args.to_dict()

        self.show_num = show_num
        self.show_page_num = show_page_num

        # 分页数量
        self.paging_num, t = divmod(self.sum_num, self.show_num)
        if t:
            self.paging_num += 1

        # 判断用户输入的页数是否超了或小了，如-10页给他定位到第1页，超出最大页数给他定位到最后一页
        # and self.paging_num 保证数据没有的情况下什么都不显示，而不是报错索引为负数。
        if self.page_num > self.paging_num and self.paging_num:
            self.page_num = self.paging_num
        elif self.page_num < 1:
            self.page_num = 1

        # 取列表值的起始值和终止值
        self.start_num = (self.page_num - 1) * self.show_num
        self.end_num = self.page_num * self.show_num

        # 页码列表
        if self.paging_num < self.show_page_num:
            self.page_list = [i for i in range(1, self.paging_num + 1)]
        else:
            if self.page_num + self.show_page_num // 2 > self.paging_num:
                self.page_list = [i for i in range(self.paging_num - self.show_page_num + 1, self.paging_num + 1)]
            elif self.page_num - self.show_page_num // 2 < 1:
                self.page_list = [i for i in range(1, self.show_page_num + 1)]
            else:
                self.page_list = [i for i in range(self.page_num - self.show_page_num // 2,
                                                   self.page_num + self.show_page_num // 2 + 1)]

        # print(self.page_list)

        # 前一页及后一页的页码
        self.prev_page_num = self.page_num - 1 if self.page_num > 1 else 1
        self.next_page_num = self.page_num + 1 if self.page_num < self.paging_num else self.page_num

    # 返回取当前页数据的起始值
    @property
    def start(self):
        return self.start_num

    # 终止值
    @property
    def end(self):
        return self.end_num

    def join_params(self, params):
        temp = []
        for k, v in params.items():
            temp.append("%s=%s" % (k, v))
        return '&'.join(temp)

    def make_label(self):
        temp = '
'

        # 前一页标签
        if self.page_num == 1:
            temp += '
«
'
        else:
            self.param["page"] = self.prev_page_num
            temp += '
«
'.format(
                self.path, self.join_params(self.param))

        # 页码
        for i in self.page_list:
            self.param["page"] = i
            if i == self.page_num:
                temp += '
{2}(current)
'.format(
                    self.path, self.join_params(self.param), i)
            else:
                temp += '
{2}
'.format(self.path, self.join_params(self.param), i)

        # 后一页标签
        if self.next_page_num == self.page_num:
            temp += '
»
'
        else:
            self.param["page"] = self.next_page_num
            temp += '
»
'.format(
                self.path, self.join_params(self.param))

        temp += '
'

        return temp


import os
import sys
import platform
import subprocess

import pytest

import virtual_environments
from service_messages import ServiceMessage, assert_service_messages, has_service_messages

def construct_fixture():
    params = []

    if sys.version_info >= (2, 6):
        # latest version
        params.append(("pytest",))

    if (2, 5) <= sys.version_info < (2, 6):
        params.append(("pytest==2.5.0", "py==1.4.19"))

    if (2, 4) <= sys.version_info < (2, 5):
        params.append(("pytest==2.3.3", "py==1.4.12"))

    @pytest.fixture(scope='module', params=params)
    def venv(request):
        return virtual_environments.prepare_virtualenv(request.param)

    return venv

globals()['venv'] = construct_fixture()

def fix_slashes(s):
    if platform.system() == 'Windows':
        return s.replace('/', '\\')
    else:
        return s.replace('\\', '/')

if (sys.version_info[0] == 2 and sys.version_info >= (2, 7)) or (sys.version_info[0] == 3 and sys.version_info >= (3, 4)):
    def test_pytest_pep8(venv):
        venv_with_pep8 = virtual_environments.prepare_virtualenv(venv.packages + ("pytest-pep8",))

        output = run(venv_with_pep8, 'pep8_test.py', options="--pep8")
        pep8_test_name = "tests.guinea-pigs.pytest.pep8_test.PEP8"
        test_name = "tests.guinea-pigs.pytest.pep8_test.test_ok"
        ms = assert_service_messages(
            output,
            [
                ServiceMessage('testStarted', {'name': pep8_test_name}),
                ServiceMessage('testFailed', {'name': pep8_test_name}),
                ServiceMessage('testFinished', {'name': pep8_test_name}),
                ServiceMessage('testStarted', {'name': test_name}),
                ServiceMessage('testFinished', {'name': test_name}),
            ])

        assert ms[1].params["details"].find("E302 expected 2 blank lines, found 1") > 0

    def test_pytest_pylint(venv):
        venv_with_pylint = virtual_environments.prepare_virtualenv(venv.packages + ("pytest-pylint",))

        output = run(venv_with_pylint, 'pylint_test.py', options="--pylint")
        pylint_test_name = "tests.guinea-pigs.pytest.pylint_test.Pylint"
        test_name = "tests.guinea-pigs.pytest.pylint_test.test_ok"
        ms = assert_service_messages(
            output,
            [
                ServiceMessage('testStarted', {'name': pylint_test_name}),
                ServiceMessage('testStdErr', {'name': pylint_test_name}),
                ServiceMessage('testFailed', {'name': pylint_test_name}),
                ServiceMessage('testFinished', {'name': pylint_test_name}),
                ServiceMessage('testStarted', {'name': test_name}),
                ServiceMessage('testFinished', {'name': test_name}),
            ])

        assert ms[1].params["out"].find("No config file found") >= 0
        assert ms[2].params["details"].find("Unused import sys") > 0

def test_hierarchy(venv):
    output = run(venv, 'namespace')
    test_name = 'tests.guinea-pigs.pytest.namespace.pig_test.TestSmoke.test_smoke'
    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test_name, 'flowId': test_name}),
            ServiceMessage('testFinished', {'name': test_name, 'flowId': test_name}),
        ])

def test_force_tc_reporting(venv):
    output = run(venv, 'namespace', options="--teamcity", set_tc_version=False)
    assert has_service_messages(output)

def test_tc_reporting(venv):
    output = run(venv, 'namespace')
    assert has_service_messages(output)

def test_no_reporting_when_no_teamcity(venv):
    output = run(venv, 'namespace', set_tc_version=False)
    assert not has_service_messages(output)

def test_reporting_disabled(venv):
    output = run(venv, 'namespace', set_tc_version=True, options="--no-teamcity")
    assert not has_service_messages(output)

def test_custom_test_items(venv):
    output = run(venv, 'custom')
    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.custom.test_simple_yml.line1'}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.custom.test_simple_yml.line1'}),
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.custom.test_simple_yml.line2'}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.custom.test_simple_yml.line2'}),
        ])

if sys.version_info >= (2, 6):
    @pytest.mark.parametrize("coverage_version, pytest_cov_version", [
        ("==3.7.1", "==1.8.1"),
        ("==4.0.1", "==2.2.0"),
        # latest
        ("", ""),
    ])
    def test_coverage(venv, coverage_version, pytest_cov_version):
        if coverage_version != "==3.7.1" and (3, 1) < sys.version_info < (3, 3):
            pytest.skip("coverage >= 4.0 dropped support for Python 3.2")

        venv_with_coverage = virtual_environments.prepare_virtualenv(
            venv.packages + (
                "coverage" + coverage_version,
                "pytest-cov" + pytest_cov_version))

        output = run(venv_with_coverage, 'coverage_test', options="--cov coverage_test")
        test_name = "tests.guinea-pigs.pytest.coverage_test.coverage_test.test_covered_func"
        assert_service_messages(
            output,
            [
                ServiceMessage('testStarted', {'name': test_name}),
                ServiceMessage('testFinished', {'name': test_name}),
                ServiceMessage('buildStatisticValue', {'key': 'CodeCoverageAbsLCovered', 'value': '9'}),
                ServiceMessage('buildStatisticValue', {'key': 'CodeCoverageAbsLTotal', 'value': '13'}),
                ServiceMessage('buildStatisticValue', {'key': 'CodeCoverageAbsLUncovered', 'value': '4'}),
            ])

def test_runtime_error(venv):
    output = run(venv, 'runtime_error_test.py')
    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.runtime_error_test.test_exception'}),
            ServiceMessage('testFailed', {'flowId': 'tests.guinea-pigs.pytest.runtime_error_test.test_exception'}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.runtime_error_test.test_exception'}),
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.runtime_error_test.test_error'}),
            ServiceMessage('testFailed', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.runtime_error_test.test_error'}),
        ])
    assert ms[1].params["details"].find("raise Exception") > 0
    assert ms[1].params["details"].find("oops") > 0
    assert ms[4].params["details"].find("assert 0 != 0") > 0

def test_unittest_error(venv):
    output = run(venv, 'unittest_error_test.py')
    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.unittest_error_test.TestErrorFail.test_error'}),
            ServiceMessage('testFailed', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.unittest_error_test.TestErrorFail.test_error'}),
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.unittest_error_test.TestErrorFail.test_fail'}),
            ServiceMessage('testFailed', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.unittest_error_test.TestErrorFail.test_fail'}),
        ])
    assert ms[1].params["details"].find("raise Exception") > 0
    assert ms[1].params["details"].find("oops") > 0
    assert ms[4].params["details"].find("AssertionError") > 0

def test_fixture_error(venv):
    output = run(venv, 'fixture_error_test.py')

    test1_name = 'tests.guinea-pigs.pytest.fixture_error_test.test_error1'
    test2_name = 'tests.guinea-pigs.pytest.fixture_error_test.test_error2'

    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test1_name, 'flowId': test1_name}),
            ServiceMessage('testFailed', {'name': test1_name,
                                          'message': 'test setup failed',
                                          'flowId': test1_name}),
            ServiceMessage('testFinished', {'name': test1_name}),

            ServiceMessage('testStarted', {'name': test2_name, 'flowId': test2_name}),
            ServiceMessage('testFailed', {'name': test2_name,
                                          'message': 'test setup failed',
                                          'flowId': test2_name}),
            ServiceMessage('testFinished', {'name': test2_name}),
        ])
    assert ms[1].params["details"].find("raise Exception") > 0
    assert ms[1].params["details"].find("oops") > 0
    assert ms[4].params["details"].find("raise Exception") > 0
    assert ms[4].params["details"].find("oops") > 0

@pytest.mark.skipif("sys.version_info < (2, 6)", reason="requires Python 2.6+")
def test_output(venv):
    output = run(venv, 'output_test.py')

    test_name = 'tests.guinea-pigs.pytest.output_test.test_out'

    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test_name, 'flowId': test_name, 'captureStandardOutput': 'false'}),
            ServiceMessage('blockOpened', {'name': 'test setup', 'flowId': test_name}),
            ServiceMessage('testStdOut', {'name': test_name, 'flowId': test_name, 'out': 'setup stdout|n'}),
            ServiceMessage('testStdErr', {'name': test_name, 'flowId': test_name, 'out': 'setup stderr|n'}),
            ServiceMessage('blockClosed', {'name': 'test setup'}),
            ServiceMessage('testStdOut', {'name': test_name, 'flowId': test_name, 'out': 'test stdout|n'}),
            ServiceMessage('testStdErr', {'name': test_name, 'flowId': test_name, 'out': 'test stderr|n'}),
            ServiceMessage('testFinished', {'name': test_name, 'flowId': test_name}),
            ServiceMessage('blockOpened', {'name': 'test teardown', 'flowId': test_name}),
            ServiceMessage('testStdOut', {'name': test_name, 'flowId': test_name, 'out': 'teardown stdout|n'}),
            ServiceMessage('testStdErr', {'name': test_name, 'flowId': test_name, 'out': 'teardown stderr|n'}),
            ServiceMessage('blockClosed', {'name': 'test teardown'}),
        ])

@pytest.mark.skipif("sys.version_info < (2, 6)", reason="requires Python 2.6+")
def test_chunked_output(venv):
    output = run(venv, 'chunked_output_test.py')

    full_line = 'x' * 50000
    leftovers = 'x' * (1024 * 1024 - 50000 * 20)

    assert_service_messages(
        output,
        [ServiceMessage('testStarted', {})] +
        [ServiceMessage('testStdOut', {'out': full_line})] * 20 +
        [ServiceMessage('testStdOut', {'out': leftovers})] +
        [ServiceMessage('testStdErr', {'out': full_line})] * 20 +
        [ServiceMessage('testStdErr', {'out': leftovers})] +
        [ServiceMessage('testFinished', {})]
    )

def test_output_no_capture(venv):
    output = run(venv, 'output_test.py', options="-s")

    test_name = 'tests.guinea-pigs.pytest.output_test.test_out'

    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test_name, 'flowId': test_name, 'captureStandardOutput': 'true'}),
            ServiceMessage('testFinished', {'name': test_name, 'flowId': test_name}),
        ])
    assert "setup stderr" in output
    assert "setup stdout" in output
    assert "test stderr" in output
    assert "test stdout" in output
    assert "teardown stderr" in output
    assert "teardown stdout" in output

def test_teardown_error(venv):
    output = run(venv, 'teardown_error_test.py')
    teardown_test_id = 'tests.guinea-pigs.pytest.teardown_error_test.test_error_teardown'
    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.teardown_error_test.test_error'}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.teardown_error_test.test_error'}),
            ServiceMessage('testStarted', {'name': teardown_test_id, 'flowId': teardown_test_id}),
            ServiceMessage('testFailed', {'flowId': teardown_test_id,
                                          'message': fix_slashes('tests/guinea-pigs/pytest/teardown_error_test.py') + ':13 (test_error)'}),
            ServiceMessage('testFinished', {'name': teardown_test_id, 'flowId': teardown_test_id}),
        ])
    assert ms[3].params["details"].find("raise Exception") > 0
    assert ms[3].params["details"].find("teardown oops") > 0

def test_module_error(venv):
    output = run(venv, 'module_error_test.py')
    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.module_error_test.top_level_collect'}),
            ServiceMessage('testFailed', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.module_error_test.top_level_collect'}),
        ])
    assert ms[1].params["details"].find("raise Exception") > 0
    assert ms[1].params["details"].find("module oops") > 0

def test_skip(venv):
    output = run(venv, 'skip_test.py')
    test_name = 'tests.guinea-pigs.pytest.skip_test.test_function'
    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test_name}),
            ServiceMessage('testIgnored', {'message': 'Skipped: skip reason', 'flowId': test_name}),
            ServiceMessage('testFinished', {'name': test_name}),
        ])

def test_params(venv):
    output = run(venv, 'params_test.py')

    test1_name = 'tests.guinea-pigs.pytest.params_test.test_eval|[3+5-8|]'
    test2_name = "tests.guinea-pigs.pytest.params_test.test_eval|[|'1_5|' + |'2|'-1_52|]"
    test3_name = 'tests.guinea-pigs.pytest.params_test.test_eval|[6*9-42|]'

    assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': test1_name}),
            ServiceMessage('testFinished', {'name': test1_name}),
            ServiceMessage('testStarted', {'name': test2_name}),
            ServiceMessage('testFinished', {'name': test2_name}),
            ServiceMessage('testStarted', {'name': test3_name}),
            ServiceMessage('testFailed', {'name': test3_name,
                                          'message': fix_slashes('tests/guinea-pigs/pytest/params_test.py') + ':3 (test_eval|[6*9-42|])'}),
            ServiceMessage('testFinished', {'name': test3_name}),
        ])

def test_xfail(venv):
    output = run(venv, 'xfail_test.py')
    ms = assert_service_messages(
        output,
        [
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.xfail_test.test_unexpectedly_passing'}),
            ServiceMessage('testFailed', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.xfail_test.test_unexpectedly_passing'}),
            ServiceMessage('testStarted', {'name': 'tests.guinea-pigs.pytest.xfail_test.test_expected_to_fail'}),
            ServiceMessage('testIgnored', {}),
            ServiceMessage('testFinished', {'name': 'tests.guinea-pigs.pytest.xfail_test.test_expected_to_fail'}),
        ])
    assert ms[4].params["message"].find("xfail reason") > 0

def run(venv, file_name, test=None, options='', set_tc_version=True):
    env = virtual_environments.get_clean_system_environment()

    if set_tc_version:
        env['TEAMCITY_VERSION'] = "0.0.0"

    if test is not None:
        test_suffix = "::" + test
    else:
        test_suffix = ""

    command = os.path.join(venv.bin, 'py.test') + " " + options + " " + \
        os.path.join('tests', 'guinea-pigs', 'pytest', file_name) + test_suffix
    print("RUN: " + command)
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, shell=True)
    output = "".join([x.decode() for x in proc.stdout.readlines()])
    print("OUTPUT: " + output.replace("#", "*"))
    proc.wait()

    return output


from random import randint
from urllib.parse import urlparse

from django.conf import settings
from django.conf.urls import url
from django.shortcuts import redirect, render
from django.utils.baseconv import base56

from .models import Url

ALLOWED_SCHEMES = {'http', 'https', 'ftp'}

def index(request):
    if request.method == 'GET':
        return render(request, 'index.html')
    if request.method == 'POST':
        u = request.POST.get('url', '')
        if urlparse(u).scheme not in ALLOWED_SCHEMES:
            msg = 'Введите коректный URL, Разрешенные схемы: {}'.format(
                ', '.join(ALLOWED_SCHEMES))
            return render(
                request, 'index.html', context={'message': msg})
        else:
            url_obj, created = Url.objects.get_or_create(source=u)
            if created:
                url_obj.key = base56.encode(randint(0, 0x7fffff))
                url_obj.save()
            short_url = 'http://localhost:8000/' + url_obj.key
            msg = 'Ваша короткая ссылка:'
            return render(
                request, 'index.html',
                context={'message': msg, 'url': short_url})

def redirect_view(request, key):
    url_obj = Url.objects.filter(key=key).first()
    if url_obj:
        url_obj.redirect_count += 1
        url_obj.save()
        return redirect(url_obj.source)
    return redirect('/')


import CoreData
from PyObjCTools.TestSupport import TestCase, min_os_level

class TestNSCoreDataCoreSpotlightDelegate(TestCase):
    @min_os_level("10.13")
    def testMethods(self):
        self.assertArgIsBlock(
            CoreData.NSCoreDataCoreSpotlightDelegate.searchableIndex_reindexSearchableItemsWithIdentifiers_acknowledgementHandler_,
            2,
            b"v",
        )


# https://www.owasp.org/index.php/List_of_useful_HTTP_headers

# Based on the above website information about http headers

# Nonces ---> need to check forms for nonce id's, requires parsing the content and identifying nonceIDS

from bs4 import BeautifulSoup

class AnalyseHeader(object):
    def __init__(self):
        self.CSP = ["content-security-policy", "x-content-security-policy", "x-webkit-csp"]
        self.XFRAME = ["x-frame-options", "frame-options"]
        self.XSS = "x-xss-protection"
        self.CSRF = "nonce"
        self.HSTS = "strict-transport-security"
        self.Cookie = "set-cookie"

    def checkCSP(self, header):
        csp_directives = ['base-uri', 'child-src', 'connect-src', 'default-src', 'font-src', 'form-action', 'frame-ancestors', 'frame-src', 'img-src', 'media-src', 'object-src', 'plugin-types', 'report-uri','script-src', 'style-src', 'upgrade-insecure-requests']
        csp_map = {'implemented' : False}
        for csp in self.CSP:
            if csp in header:
                csp_map['implemented'] = True
                for directive in csp_directives:
                    csp_map[directive] = False
                policy_string = header[csp]
                policy_list = policy_string.split(';')
                for policy in policy_list:
                    policy_strings = policy.strip().split(' ')
                    if policy_strings[0] in csp_directives:
                        csp_map[policy_strings[0]] = True
        return csp_map

    def checkHSTS(self, header):
        hsts_directives = ['max-age', 'includesubdomains', 'preload']
        hsts_map = {'implemented' : False}
        if self.HSTS in header:
            hsts_map['implemented'] = True
            for directive in hsts_directives:
                hsts_map[directive] = False
            policy_string = header[self.HSTS]
            policy_list = policy_string.split(';')
            for policy in policy_list:
                policy_strings = policy.strip().split('=')
                if policy_strings[0].lower() in hsts_directives:
                    if policy_strings[0].lower() == hsts_directives[0]:
                        hsts_map[policy_strings[0].lower()] = policy_strings[1].lower()
                    else:
                        hsts_map[policy_strings[0].lower()] = True
        return hsts_map

    def checkXFrame(self, header):
        xframe_modes = ['deny', 'sameorigin', 'allow-from']
        xframe_map = {'implemented' : False}
        for xframe in self.XFRAME:
            if xframe in header:
                xframe_map['implemented'] = True
                policy_string = header[xframe]
                policy_list = policy_string.split(';')
                policy_strings = policy_list[0].split(' ')
                if policy_strings[0].lower() in xframe_modes:
                    xframe_map['mode'] = policy_strings[0].lower()
                if policy_strings[0] == 'allow-from':
                    assert len(policy_strings) == 2, "Need an external framing page URI for 'allow-from' framing mode"
                    xframe_map['framing_page_uri'] = policy_strings[1]
        return xframe_map

    def checkXSS(self, header):
        xss_directives = ['mode']
        xss_map = {'implemented' : False}
        if self.XSS in header:
            for directive in xss_directives:
                xss_map[directive] = False
            policy_string = header[self.XSS]
            policy_list = policy_string.split(';')
            for policy in policy_list:
                policy_strings = policy.strip().split('=')
                if policy_strings[0] == '1':
                    xss_map['implemented'] = True
                elif policy_strings[0] == xss_directives[0]:
                    if policy_strings[1] == 'block':
                        xss_map['mode'] = 'block'
                    else:
                        xss_map['mode'] = 'sanitize'
        self.checkSecureCookie(header, xss_map)
        return xss_map

    def checkCSRF(self, header, content1, content2):
        nonce_fields = ['appActionToken', 'secTok', 'authenticity_token', 'nonce', 'data-form-nonce']
        csrf_map = {'implemented' : False}
        # Check for nonce in the header
        for each in self.CSP:
            if each in header and self.CSRF in header[each]:
                csrf_map['implemented'] = True
                return csrf_map
        # Check for nonce in the forms
        soup1 = BeautifulSoup(content1, "lxml")
        soup2 = BeautifulSoup(content2, "lxml")
        forms1 = soup1.find_all('form')
        forms2 = soup2.find_all('form')
        for form1,form2 in zip(range(len(forms1)), range(len(forms2))):
            form1_attrs = forms1[form1].attrs
            form2_attrs = forms2[form2].attrs
            # Check whether form attribute has a nonce and value is differnet for multiple requests
            for form_attr in form1_attrs:
                if self.CSRF in form_attr:
                    if form1_attrs[form_attr] != form2_attrs[form_attr]:
                        csrf_map['implemented'] = True
                        return csrf_map
            # Checks whether input tag has a nonce field and value is different for multiple requests
            form1_inputtags = forms1[form1].find_all('input')
            form2_inputtags = forms2[form2].find_all('input')
            for inputtag in range(len(form1_inputtags)):
                form1_inputtag_attrs = form1_inputtags[inputtag].attrs
                form2_inputtag_attrs = form2_inputtags[inputtag].attrs
                if 'type' in form1_inputtag_attrs and form1_inputtag_attrs['type']=='hidden':
                    if 'name' in form1_inputtag_attrs and form1_inputtag_attrs['name'] in nonce_fields:
                        if form1_inputtag_attrs.get('value') != form2_inputtag_attrs.get('value'):
                            csrf_map['implemented'] = True
                            return csrf_map
        return csrf_map

    def checkSecureCookie(self, header, xss_map):
        xss_map['httpOnly'] = False
        xss_map['secure'] = False
        if self.Cookie in header:
            compString = header[self.Cookie].lower()
            if 'secure' in compString:
                xss_map['secure'] = True
            if 'httponly' in compString:
                xss_map['httpOnly'] = True

    def checkURLS(self, header, content1, content2):
        vul = {}
        vul['csp'] = self.checkCSP(header)
        vul['csrf'] = self.checkCSRF(header, content1, content2)
        vul['hsts'] = self.checkHSTS(header)
        vul['xframe'] = self.checkXFrame(header)
        vul['xss'] = self.checkXSS(header)
        return vul

'''
a = AnalyseHeader()
import httplib2
import json
h = httplib2.Http(".cache")
(header1, content1) = h.request("https://www.airbnb.com", "GET")
(header2, content2) = h.request("https://www.airbnb.com", "GET")
print json.dumps(header1, indent=4, sort_keys=True)
print json.dumps(a.checkURLS(header1, content1, content2), indent=4, sort_keys=True)'''


#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Mon Jul 22 19:56:11 CEST 2013
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import bob.io.base
import bob.ip.color
import numpy

def atnt_database_directory(atnt_user_directory = None):
  #Checks, where the AT&T database is located and downloads it on need
  if atnt_user_directory is not None:
    # a user directory is specified
    atnt_default_directory = atnt_user_directory
  elif 'ATNT_DATABASE_DIRECTORY' in os.environ:
    # the environment variable is set
    atnt_default_directory = os.environ['ATNT_DATABASE_DIRECTORY']
  else:
    atnt_default_directory = 'Database'

  # Check if the database is already in the specified directory
  if os.path.isdir(atnt_default_directory) and set(['s%d'%s for s in range(1,41)]).issubset(os.listdir(atnt_default_directory)):
    return atnt_default_directory

  # ... download the database ...

  # create directory
  if not os.path.exists(atnt_default_directory):
    os.mkdir(atnt_default_directory)

  # setup
  import sys, tempfile
  if sys.version_info[0] <= 2:
    import urllib2 as urllib
  else:
    import urllib.request as urllib

  db_url = 'http://www.cl.cam.ac.uk/Research/DTG/attarchive/pub/data/att_faces.zip'
  import logging
  logger = logging.getLogger('bob')
  logger.warn("Downloading the AT&T database from '%s' to '%s' ..." % (db_url, atnt_default_directory))

  # download
  url = urllib.urlopen(db_url)
  local_zip_file = tempfile.mkstemp(prefix='atnt_db_', suffix='.zip')[1]
  dfile = open(local_zip_file, 'wb')
  dfile.write(url.read())
  dfile.close()

  # unzip
  import zipfile
  zip = zipfile.ZipFile(local_zip_file)
  zip.extractall(atnt_default_directory)
  zip.close()

  # remove temporary zip file
  os.remove(local_zip_file)

  return atnt_default_directory

def load_images(database, files, face_cropper = None, preprocessor = None):
  """Loads the original images for the given list of File objects.

  Keyword Parameters:

  database : :py:class:`bob.db.verification.utils.Database` or derived from it
    The database interface to query.

  files : [:py:class:`bob.db.verification.utils.File`] or derived from it
    A list of ``File`` objects for which the images should be loaded.

  face_cropper : :py:class:`bob.ip.base.FaceEyesNorm` or ``None``
    If a face cropper is given, the face will be cropped using the eye annotations of the database.

  preprocessor : ``function``, ``object`` or ``None``
    If a preprocessor is given, it will be applied to the (cropped) faces.
  """
  file_names = database.original_file_names(files)
  # iterate through the list of file names and load images
  images = [bob.io.base.load(file_name) for file_name in file_names]
  # convert color to gray images if necessary
  gray_images = []
  for image in images:
    gray_image = bob.ip.color.rgb_to_gray(image).astype(numpy.float64) if len(image.shape) == 3 else image
    gray_images.append(gray_image.astype(numpy.float64))

  if face_cropper is not None:
    cropped_images = []
    for i in range(len(files)):
      # get the annotations for the files
      annotations = database.annotations(files[i])
      assert 'reye' in annotations and 'leye' in annotations
      cropped_images.append(face_cropper(gray_images[i], right_eye = annotations['reye'], left_eye = annotations['leye']))
  else:
    cropped_images = gray_images

  if preprocessor is not None:
    preprocessed_images = [preprocessor(i) for i in cropped_images]
  else:
    preprocessed_images = cropped_images

  return preprocessed_images


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit as cf

class Mobility:
    DataAddress = ""
    sigma = 0.01
    title = ""
    x_label = "B-Field"
    x_units = "Teslas"
    y_label_nonlinear = "Rxx"
    y_label_linear = "Rxy"
    y_units = "Ohms"
    guess_xx = []
    guess_xy = []
    params_xx = []
    params_xy = []

    def __init__(self, address, guesscoefficientxx, guessoffsetxx, guesscoefficientxy, guessoffsetxy):
        self.DataAddress = address[:-8] + "_Rxx" + address[-8:]
        self.guess_xx = [guesscoefficientxx, guessoffsetxx]
        self.guess_xy = [guesscoefficientxy, guessoffsetxy]
        self.title = address[90:99]

    @staticmethod
    def linearfunc(bfield, coefficient, offset):
        return coefficient*bfield+offset

    @staticmethod
    def parabolicfunc(bfield, coefficient, offset):
        return coefficient*(bfield**2.0)+offset

    def chisquared(self, params, bfield, rxy):
        return np.sum((rxy - self.linearfunc(bfield, *params)) ** 2 / self.sigma ** 2)

    def pltinitial(self, data, xx, linearplot):
        bfieldguess = np.linspace(min(data[1, :]), max(data[1, :]), 500)
        if linearplot:
            rguess = self.linearfunc(bfieldguess, *self.guess_xy)
        else:
            rguess = self.parabolicfunc(bfieldguess, *self.guess_xx)

        plt.scatter(data[1, :], data[0, :], marker='.', label="measured data")
        plt.plot(bfieldguess, rguess, marker="", linestyle="-", linewidth=1, color="g", label="initial guess")
        plt.xlabel('{} [{}]'.format(self.x_label, self.x_units))
        if xx:
            plt.ylabel('{} [{}]'.format(self.y_label_nonlinear, self.y_units))
            plt.title(self.title + " Rxx Initial Guess")
        else:
            plt.ylabel('{} [{}]'.format(self.y_label_linear, self.y_units))
            plt.title(self.title + " Rxy Initial Guess")
        plt.legend(loc='best', numpoints=1)
        print('\nSaving plot 1 for ' + self.title)
        if "OutPlane" in self.DataAddress:
            if xx:
                address = self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxx_Initial" + ".png"
            else:
                address = self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxy_Initial" + ".png"
        elif xx:
            address = self.DataAddress[:90] + "Graphs/" + self.title + "_Rxx_Initial" + ".png"
        else:
            address = self.DataAddress[:90] + "Graphs/" + self.title + "_Rxy_Initial" + ".png"
        plt.savefig(address)
        plt.clf()

    def pltfit(self, data, xx, linearplot):
        if linearplot:
            self.params_xy, cov = cf(self.linearfunc, data[1, :], data[0, :],
                                     sigma=self.sigma * np.ones(len(data[1, :])), p0=self.guess_xy, maxfev=10 ** 5)
            chi2 = self.chisquared(self.params_xy, data[1, :], data[0, :])
            dof = len(data[1, :]) - len(self.params_xy)
        else:
            self.params_xx, cov = cf(self.parabolicfunc, data[1, :], data[0, :],
                                     sigma=self.sigma * np.ones(len(data[1, :])), p0=self.guess_xx, maxfev=10 ** 5)
            chi2 = self.chisquared(self.params_xx, data[1, :], data[0, :])
            dof = len(data[1, :]) - len(self.params_xx)

        print("\nGoodness of fit - chi square measure:")
        print("Chi2 = {}, Chi2/dof = {}\n".format(chi2, chi2 / dof))

        cov = cov * dof / chi2
        paramserr = np.sqrt(np.diag(cov))

        print("Fit parameters:")
        if linearplot:
            param_names = ['Slope', 'Offset']
            for i in range(len(self.params_xy)):
                print('{} = {:.3e} +/- {:.3e}'.format(param_names[i], self.params_xy[i], paramserr[i]))
            bfieldfit = np.linspace(min(data[1, :]), max(data[1, :]), len(data[1, :]) * 10)
            rxyfit = self.linearfunc(bfieldfit, *self.params_xy)
        else:
            param_names = ['Coefficient', 'Offset']
            for i in range(len(self.params_xx)):
                print('{} = {:.3e} +/- {:.3e}'.format(param_names[i], self.params_xx[i], paramserr[i]))
            bfieldfit = np.linspace(min(data[1, :]), max(data[1, :]), len(data[1, :]) * 10)
            rxyfit = self.parabolicfunc(bfieldfit, *self.params_xx)

        plt.scatter(data[1, :], data[0, :], marker='.', label="measured data")
        plt.plot(bfieldfit, rxyfit, marker="", linestyle="-", linewidth=2, color="r", label=" fit")
        plt.xlabel('{} [{}]'.format(self.x_label, self.x_units))
        if xx:
            plt.ylabel('{} [{}]'.format(self.y_label_nonlinear, self.y_units))
            plt.title(self.title + " Rxx Fitted Guess")
        else:
            plt.ylabel('{} [{}]'.format(self.y_label_linear, self.y_units))
            plt.title(self.title + " Rxy Fitted Guess")
        plt.legend(loc='best', numpoints=1)
        print('\nSaving plot 2 for ' + self.title)
        if "OutPlane" in self.DataAddress:
            if xx:
                address = self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxx_Fitted" + ".png"
            else:
                address = self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxy_Fitted" + ".png"
        elif xx:
            address = self.DataAddress[:90] + "Graphs/" + self.title + "_Rxx_Fitted" + ".png"
        else:
            address = self.DataAddress[:90] + "Graphs/" + self.title + "_Rxy_Fitted" + ".png"
        plt.savefig(address)
        plt.clf()

    def pltresiduals(self, data, xx, linearplot):
        if linearplot:
            rfit = self.linearfunc(data[1, :], *self.params_xy)
        else:
            rfit = self.parabolicfunc(data[1, :], *self.params_xx)
        residual = data[0, :] - rfit

        if xx:
            plt.title(self.title + " Rxx Residuals")
        else:
            plt.title(self.title + " Rxy Residuals")
        plt.scatter(data[1, :], residual, marker='.', label="residual (y-y_fit)")
        plt.hlines(0, np.min(data[1, :]), np.max(data[1, :]), lw=2, alpha=0.8)
        plt.xlabel('{} [{}]'.format(self.x_label, self.x_units))
        plt.ylabel('y-y_fit [{}]'.format(self.y_units))
        plt.legend(loc='best', numpoints=1)

        print('\nSaving plot 3 for ' + self.title)
        if "OutPlane" in self.DataAddress:
            if xx:
                plt.savefig(self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxx_Residuals" + ".png")
            else:
                plt.savefig(self.DataAddress[:90] + "Graphs/" + self.title + "_InPlane_Rxy_Residuals" + ".png")
        elif xx:
            plt.savefig(self.DataAddress[:90] + "Graphs/" + self.title + "_Rxx_Residuals" + ".png")
        else:
            plt.savefig(self.DataAddress[:90] + "Graphs/" + self.title + "_Rxy_Residuals" + ".png")
        plt.clf()

    def main(self, rxyzero):
        rxxdata = np.loadtxt(self.DataAddress)
        address = list(self.DataAddress)
        address[-9] = 'y'
        rxyinit = np.loadtxt("".join(address))

        if self.title == "PST236 R5":
            rxxdataprime = rxxdata[:, np.where(np.abs(rxxdata[1]) < 1)][:, 0, :]
        elif not self.title == "PST380 I2":
            rxxdataprime = rxxdata[:, np.where(np.abs(rxxdata[1]) > 0.2)][:, 0, :]
            rxxdataprime = rxxdataprime[:, np.where(np.abs(rxxdataprime[1]) < 3)][:, 0, :]
        else:
            rxxdataprime = rxxdata

        self.pltinitial(rxxdataprime, True, False)
        self.pltfit(rxxdataprime, True, False)
        self.pltresiduals(rxxdataprime, True, False)

        rxyprime = rxyinit - (rxyzero / self.parabolicfunc(0, *self.params_xx)) * rxxdata

        self.pltinitial(rxyprime, False, True)
        self.pltfit(rxyprime, False, True)
        self.pltresiduals(rxyprime, False, True)

        return [self.params_xx, self.params_xy]

EP3/Mobility and Carrier Density/MobilityClass.py


# -*- coding: utf-8 -*-
import json

from django.core import mail

from olympia.amo.tests import TestCase
from olympia.addons.models import Addon
from olympia.stats.models import Contribution
from olympia.stats.db import StatsDictField
from olympia.users.models import UserProfile

class TestStatsDictField(TestCase):

    def test_to_python_none(self):
        assert StatsDictField().to_python(None) is None

    def test_to_python_dict(self):
        assert StatsDictField().to_python({'a': 1}) == {'a': 1}

    def test_to_python_json(self):
        val = {'a': 1}
        assert StatsDictField().to_python(json.dumps(val)) == val

class TestEmail(TestCase):
    fixtures = ['base/users', 'base/addon_3615']

    def setUp(self):
        super(TestEmail, self).setUp()
        self.addon = Addon.objects.get(pk=3615)
        self.user = UserProfile.objects.get(pk=999)

    def make_contribution(self, amount, locale):
        return Contribution.objects.create(addon=self.addon, amount=amount,
                                           source_locale=locale)

    def test_thankyou_note(self):
        self.addon.enable_thankyou = True
        self.addon.thankyou_note = u'Thank "quoted". <script>'
        self.addon.name = u'Test'
        self.addon.save()
        cont = self.make_contribution('10', 'en-US')
        cont.update(transaction_id='yo',
                    post_data={'payer_email': 'test@tester.com'})

        cont.mail_thankyou()
        assert len(mail.outbox) == 1
        email = mail.outbox[0]
        assert email.to == ['test@tester.com']
        assert '"' not in email.body
        assert u'Thank "quoted".' in email.body
        assert '<script>' not in email.body
        assert '<script>' not in email.body


#!/usr/bin/env python3.3
# -*- coding: utf8 -*-

from pprint import pprint as prn # < DEBUG
import re
import logging
import inspect
import threading
import os.path
from abc import ABCMeta
from abc import abstractmethod
from collections import deque
from collections import Sequence
from collections import MutableSequence
from multiprocessing import cpu_count
from concurrent import futures
from urllib import request
from urllib.error import URLError
from lxml import etree

_MAX_THREADS = 8

def threads_count ():
    try:
        return cpu_count()
    except NotImplementedError:
        return _MAX_THREADS

class Poller (object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__ (self, pool):
        self.__pool = pool
        self.__lock = threading.RLock()
        self.__pendings = set()

    @property
    def pendings (self):
        with self.__lock:
            return self.__pendings

    def __discard_pending (self, future):
        with self.__lock:
            self.__pendings.discard(future)

    @abstractmethod
    def _actor (self, *args, **kwargs):
        pass

    def run (self, *args, **kwargs):
        with self.__lock:
            future = self.__pool.submit(self._actor, *args, **kwargs)
            self.__pendings.add(future)
            future.add_done_callback(self.__discard_pending)
            return future

    @abstractmethod
    def result (self, *args, **kwargs):
        pass

class Fetcher (Poller):
    _log = logging.getLogger('Fetcher')

    def __init__ (self, pool):
        super().__init__(pool)
        self.__out = {}

    def _actor (self, url, path):
        """path can be a single value or a sequence of values."""
        # Create XPath objects.
        if isinstance(path, str):
            path = (path,)
        if isinstance(path, (Sequence, MutableSequence)):
            try:
                xpath = [etree.XPath(p) for p in path]
            except etree.XPathError:
                self._log.error('XPath error: "%s"', path)
                return False
        else:
            return False
        # Open URL.
        try:
            response = request.urlopen(url)
        except URLError:
            self._log.error('Can\'t open URL: "%s"', url)
            return False
        # Get elements.
        tree = etree.parse(response, etree.HTMLParser())
        h = hash((url, tuple(path)))
        self.__out[h] = [xp(tree) for xp in xpath]
        return h

    def result (self, h=None):
        if h:
            return self.__out[h]
        else:
            return self.__out

    def clear (self):
        self.__out.clear()

class App (object):
    _log = logging.getLogger('App')

    def __init__ (self, root=None, categories=None, resolutions=None,
                  output='.'):
        self.__pool = futures.ThreadPoolExecutor(threads_count())
        self.__fetcher = Fetcher(self.__pool)
        self.root = root
        self.categories = categories
        self.resolutions = resolutions
        self.output = output

    def __verify_input (self, tpl, regex):
        if isinstance(tpl, tuple):
            rx = re.compile(regex)
            return all((isinstance(s, str) and rx.match(s) for s in tpl))
        return False

    def __getter (self, caller):
        return getattr(self, '_{0}__{1}'
                       .format(self.__class__.__name__, caller),
                       None)
    def __setter (self, caller, tpl, regex=r'^.*$'):
        if self.__verify_input(tpl, regex):
            setattr(self, '_{0}__{1}'.format(self.__class__.__name__, caller),
                    tpl)
            return True
        return False

    @property
    def root (self):
        return self.__getter(inspect.stack()[0][3])
    @root.setter
    def root (self, root):
        if isinstance(root, str) and len(root.strip()) > 0:
            self.__root = root
        else:
            self._log.error('URL must be a non-empty string')

    @property
    def categories (self):
        return self.__getter(inspect.stack()[0][3])
    @categories.setter
    def categories (self, ctgs):
        """ "*" passed as a category name tells to fetch all subcategories for
        all of the categories that available on site."""
        if not self.__setter(inspect.stack()[0][3], ctgs,
                             r'^(?:\*|(?:\w+|\w+\s+)+)$'):
            self._log.error('Categories must be represented as a tuple of '
                            'string(s)')

    @property
    def resolutions (self):
        return self.__getter(inspect.stack()[0][3])
    @resolutions.setter
    def resolutions (self, resl):
        if not self.__setter(inspect.stack()[0][3], resl, r'^\d+[xX]\d+$'):
            self._log.error('Resolutions must be represented as a tuple of '
                            'string(s)')

    @property
    def output (self):
        return self.__getter(inspect.stack()[0][3])
    @output.setter
    def output (self, path):
        if isinstance(path, str) and os.path.exists(path):
            self.__output = path
        else:
            self._log.error('Given path "%s" didn\'t exists', path)

    @staticmethod
    def __pretty (iterable):
        return (x.rsplit(' ', 1)[0] for x in iterable)

    @staticmethod
    def __max_page (iterable):
        return max([int(p.rsplit('/', 1)[-1]) for p in iterable])

    @staticmethod
    def __updkey ():
        pass

    def __send_to_pool (self, mapping, path, dct, *functors):
        i = 1
        ff = {}
        page = 1
        total = 0
        is_url = False
        for c in mapping:
            if isinstance(mapping[c], str):
                ff[self.__fetcher.run(self.root + mapping[c], path)] = c
            elif isinstance(mapping[c], (Sequence, MutableSequence)):
                is_url = True
                total += len(mapping[c])
                for url in mapping[c]:
                    ff[self.__fetcher.run(self.root + url, path)] = c
            else:
                return None
        for f in futures.as_completed(ff):
            if not is_url:
                self._log.info('"%s" was successfully fetched', mapping[ff[f]])
            else:
                self._log.info('Page was successfully fetched (%d/%d)', page,
                               total)
                page += 1
            results = self.__fetcher.result(f.result())
            if 1 == len(functors):
                if self.__updkey is functors[0]:
                    dct[(url, i)] = results[0]
                    i += 1
                else:
                    dct[ff[f]] = functors[0](results[0]) if results[0] else 1
            else:
                if 2 == len(functors):
                    _ = dict
                if len(functors) > 2:
                    _ = list
                for i in range(len(functors)):
                    results[i] = tuple(functors[i](results[i]))
                    x = _(zip(*results))
                    if not is_url:
                        x = tuple([x, mapping[ff[f]]])
                    if is_url and ff[f] in dct:
                        dct[ff[f]].update(x)
                    else:
                        dct[ff[f]] = x

    def get_categories (self, show=False):
        """Fetch all of the categories and its subcategories."""
        self.__fetcher.clear()
        path = '//ul[@class="side-panel categories"]/li{0}/a'
        # Root categories.
        self._log.info('Fetching root categories for "%s"...', self.root)
        fut = self.__fetcher.run(self.root, [path.format('') + '/@title',
                                             path.format('') + '/@href'])
        futures.wait(self.__fetcher.pendings)
        res = self.__fetcher.result(fut.result())
        kategories = dict(zip(self.__pretty(res[0]), res[1]))
        # Subcategories.
        if '*' in self.categories:
            self.categories = tuple(kategories.keys())
            sel_ctgs = kategories
        if not all(c in kategories for c in self.categories):
            self._log.error('Unknown category(ies)')
            return None
        else:
            sel_ctgs = {k: kategories[k] for k in self.categories}
        path = path.format('[@style="padding-left:5px;"]')
        subs = {}
        self._log.info('Fetching subcategories...')
        self.__send_to_pool(sel_ctgs, [path + '/@title', path + '/@href'],
                            subs, self.__pretty, lambda x: x)
        if show:
            for cat in subs:
                if len(subs[cat][0]) > 0:
                    self._log.info('Subcategories for category "%s" ==>', cat)
                    for subcat in subs[cat][0]:
                        self._log.info('Sub: "%s"', subcat)
                    self._log.info('"%s" |', cat)
                else:
                    self._log.info('Category: "%s" ==>|', cat)
        return subs

    def get_wallpapers (self, subcategories):
        """Download wallpapers according to its resolution and category."""
        if not subcategories or len(subcategories) < 1:
            return None
        s = set(subcategories)
        flat_ctgs = {}
        for cat in self.categories:
            if cat in s:
                flat_ctgs[cat] = subcategories[cat][1]
                s.discard(cat)
        ss = {}
        for c in s:
            ss.update(subcategories[c][0])
        for cat in self.categories:
            if cat in ss:
                flat_ctgs[cat] = ss[cat]
        self.__fetcher.clear()
        path = '//div[@class="pagination"]/a[contains(@href, "page/")]/@href'
        pages = {}
        self._log.info('Fetching pages for an each category...')
        self.__send_to_pool(flat_ctgs, path, pages, self.__max_page)
        wallpapers = {k: [flat_ctgs[k].rsplit('.', 1)[0] + '/page/' + str(p)
            if p > 1 else flat_ctgs[k].rsplit('.', 1)[0]
            for p in range(1, pages[k] + 1)] for k in flat_ctgs}
        path = ('//li[@class="wall"]/div[@class="thumb"]/div[@class="mini-hud"]'
                '/a')
        walls = {}
        self._log.info('Fetching wallpapers by pages...')
        self.__send_to_pool(wallpapers, [path + '/@href', path + '/@title'],
                            walls, lambda x: x, lambda x: x)
        flat_walls = {}
        for c in walls:
            flat_walls[c] = list(walls[c])
        r = deque(self.resolutions)
        res = ' and ('
        try:
            while True:
                res += 'contains(@href, "-{0}.jpg")'.format(r.popleft())
                if len(r):
                    res += ' or '
        except IndexError:
            res += ')'
        path = ('//div[@class="wallpaper-resolutions"]/a[@target="_self"{0}]'
                '/@href'.format(res))
        resolutions = {}
        self._log.info('Fetching wallpaper resolutions...')
        self.__send_to_pool(flat_walls, path, resolutions, self.__updkey)
        return resolutions

def main ():
    logging.basicConfig(level=logging.INFO)

    app = App('http://wallpaperswide.com/', ('Charity', 'Motors',), ('2560x1440',))

    # DEBUG ==>
    subs = app.get_categories(True)
    # app.categories = ('ATV',)
    prn(app.get_wallpapers(subs))

if '__main__' == __name__:
    main()


class Creature:
    def __init__(self, name, terror_rating):
        self.name = name
        self.terror_rating = int(terror_rating)
        self.item_ls = []
        self.times = 1

    def take(self, item):
        self.item_ls.append(item)
        self.location.item_ls.remove(item)
        self.terror_rating += item.terror_rating

    def drop(self, item):
        self.item_ls.remove(item)
        self.location.item_ls.append(item)
        self.terror_rating -= item.terror_rating

    def get_terror_rating(self):
        return self.terror_rating

    def can_leave(self, direct):
        i = 0
        while i < len(self.location.other_loc):
            if self.location.other_loc[i][0].lower() == direct.lower():
                return self.location.other_loc[i][1]
            i += 1
        return False

    def change_direc(self):
        direct_ls = ['north', 'northeast', 'east', 'southeast', 'south', 'southwest',\
                'west', 'northwest']
        i = 0
        while i < len(direct_ls):
            if self.direct == direct_ls[i]:
                break
            i += 1
        j = i
        i = 0
        while i < 8:
            loc = self.can_leave(direct_ls[j])
            if loc:
                self.direct = direct_ls[j]
                return loc
            j = (j+1) % 8
            i += 1

    def goto(self, loc):
        self.location.crea_ls.remove(self)
        self.location = loc
        self.location.crea_ls.append(self)
        self.times = 1