"""Logging implemetation."""

import logging

def get_logger(name='cachetoolz') -> logging.Logger:
    """Get a logger.

    Parameters
    ----------
    name
        logger name

    """
    logger = logging.getLogger(name)

    log_level = logger.level
    if log_level == logging.NOTSET:
        log_level = logging.WARN
        logger.setLevel(log_level)

    formatter = logging.Formatter(
        '%(asctime)s | %(name)s | %(levelname)s | '
        '%(pathname)s:%(funcName)s:%(lineno)d - %(message)s'
    )

    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(log_level)
    stream_handler.setFormatter(formatter)

    logger.addHandler(stream_handler)

    return logger


from flask import Blueprint, render_template, session, jsonify
from .code_block import CodeBlock, DBCodeBlock

# Create two blueprints for the lobby and code_block routes
lobby_blueprint = Blueprint("lobby", __name__)
code_block_blueprint = Blueprint("code_block", __name__)

# A global variable to track if a mentor has been assigned yet
mentor_assigned = False

# The lobby route renders the lobby.html template with a list of all code blocks
@lobby_blueprint.route("/lobby")
def lobby_route():
    validate_mentor()
    user_role = session.get("role")
    data = DBCodeBlock.query.all()
    return render_template("lobby.html", code_blocks=data, user_role=user_role)

# The code_block_route renders the code_block.html template with the code block with the given id
@code_block_blueprint.route("/code-block/<id>")
def code_block_route(id):
    validate_mentor()
    user_role = session.get("role")
    from app import db
    code_block_from_db = db.get_or_404(DBCodeBlock, id)
    return render_template("code-block.html", code_block=code_block_from_db, user_role=user_role)

# The get_code_blocks route returns a JSON list of all code blocks
@lobby_blueprint.route("/get-code-blocks")
def get_code_blocks():
    validate_mentor()
    user_role = session.get("role")
    data = DBCodeBlock.query.all()
    res = []
    for cb in data:
        res.append(cb.to_dict())
    print(res)
    return jsonify(res)

# The get_code_block_by_id route returns a JSON representation of the code block with the given id
@code_block_blueprint.route("/get-code-block/<id>")
def get_code_block_by_id(id):
    validate_mentor()
    user_role = session.get("role")
    from app import db
    code_block_from_db = db.get_or_404(DBCodeBlock, id)
    return code_block_from_db.to_dict()

# The get_user_role route returns the user's role as a JSON object
@lobby_blueprint.route("/get-user-role")
def get_user_role():
    validate_mentor()
    user_role = session.get("role")
    return {'user_role': user_role}

# The validate_mentor function checks if the user has been assigned a role yet. If not, it assigns the user the role of "mentor".
def validate_mentor():
    global mentor_assigned
    user_role = session.get("role")
    if user_role is None:
        if mentor_assigned:
            session["role"] = "student"
        else:
            session["role"] = "mentor"
            mentor_assigned = True
    elif user_role == "mentor" and not mentor_assigned:
        mentor_assigned = True


import os
import cv2
import yaml
import numpy as np

import torch
import torch.nn.functional as F
from utils.utils import load_yaml, im_to_torch, get_subwindow_tracking, make_scale_pyramid, python2round

class Ocean(object):
    def __init__(self, info):
        super(Ocean, self).__init__()
        self.info = info   # model and benchmark info
        self.stride = 8
        self.align = info.align
        self.online = info.online
        self.trt = info.TRT

    def init(self, im, target_pos, target_sz, model, hp=None):
        # in: whether input infrared image
        state = dict()
        # epoch test
        p = OceanConfig()

        state['im_h'] = im.shape[0]
        state['im_w'] = im.shape[1]

        # single test
        if not hp and not self.info.epoch_test:
            prefix = [x for x in ['OTB', 'VOT', 'GOT10K', 'LASOT'] if x in self.info.dataset]
            if len(prefix) == 0: prefix = [self.info.dataset]
            absPath = os.path.abspath(os.path.dirname(__file__))
            yname = 'Ocean.yaml'
            yamlPath = os.path.join(absPath, '../../experiments/test/{0}/'.format(prefix[0]), yname)
            cfg = load_yaml(yamlPath)
            if self.online:
                temp = self.info.dataset + 'ON'
                cfg_benchmark = cfg[temp]
            else:
                cfg_benchmark = cfg[self.info.dataset]
            p.update(cfg_benchmark)
            p.renew()

            if ((target_sz[0] * target_sz[1]) / float(state['im_h'] * state['im_w'])) < 0.004:
                p.instance_size = cfg_benchmark['big_sz']
                p.renew()
            else:
                p.instance_size = cfg_benchmark['small_sz']
                p.renew()

        # double check
        # print('======= hyper-parameters: penalty_k: {}, wi: {}, lr: {}, ratio: {}, instance_sz: {}, score_sz: {} ======='.format(p.penalty_k, p.window_influence, p.lr, p.ratio, p.instance_size, p.score_size))

        # param tune
        if hp:
            p.update(hp)
            p.renew()

            # for small object (from DaSiamRPN released)
            if ((target_sz[0] * target_sz[1]) / float(state['im_h'] * state['im_w'])) < 0.004:
                p.instance_size = hp['big_sz']
                p.renew()
            else:
                p.instance_size = hp['small_sz']
                p.renew()

        if self.trt:
            print('====> TRT version testing: only support 255 input, the hyper-param is random <====')
            p.instance_size = 255
            p.renew()

        self.grids(p)   # self.grid_to_search_x, self.grid_to_search_y

        net = model

        wc_z = target_sz[0] + p.context_amount * sum(target_sz)
        hc_z = target_sz[1] + p.context_amount * sum(target_sz)
        s_z = round(np.sqrt(wc_z * hc_z))

        avg_chans = np.mean(im, axis=(0, 1))
        z_crop, _ = get_subwindow_tracking(im, target_pos, p.exemplar_size, s_z, avg_chans)

        z = z_crop.unsqueeze(0)
        net.template(z.cuda())

        if p.windowing == 'cosine':
            window = np.outer(np.hanning(p.score_size), np.hanning(p.score_size))  # [17,17]
        elif p.windowing == 'uniform':
            window = np.ones(int(p.score_size), int(p.score_size))

        state['p'] = p
        state['net'] = net
        state['avg_chans'] = avg_chans
        state['window'] = window
        state['target_pos'] = target_pos
        state['target_sz'] = target_sz

        return state

    def update(self, net, x_crops, target_pos, target_sz, window, scale_z, p):

        if self.align:
            cls_score, bbox_pred, cls_align = net.track(x_crops)

            cls_score = F.sigmoid(cls_score).squeeze().cpu().data.numpy()
            cls_align = F.sigmoid(cls_align).squeeze().cpu().data.numpy()
            cls_score = p.ratio * cls_score + (1- p.ratio) * cls_align

        else:
            cls_score, bbox_pred = net.track(x_crops)
            cls_score = F.sigmoid(cls_score).squeeze().cpu().data.numpy()

        # bbox to real predict
        bbox_pred = bbox_pred.squeeze().cpu().data.numpy()

        pred_x1 = self.grid_to_search_x - bbox_pred[0, ...]
        pred_y1 = self.grid_to_search_y - bbox_pred[1, ...]
        pred_x2 = self.grid_to_search_x + bbox_pred[2, ...]
        pred_y2 = self.grid_to_search_y + bbox_pred[3, ...]

        # size penalty
        s_c = self.change(self.sz(pred_x2-pred_x1, pred_y2-pred_y1) / (self.sz_wh(target_sz)))  # scale penalty
        r_c = self.change((target_sz[0] / target_sz[1]) / ((pred_x2-pred_x1) / (pred_y2-pred_y1)))  # ratio penalty

        penalty = np.exp(-(r_c * s_c - 1) * p.penalty_k)
        pscore = penalty * cls_score

        # window penalty
        pscore = pscore * (1 - p.window_influence) + window * p.window_influence

        if self.online_score is not None:
            s_size = pscore.shape[0]
            o_score = cv2.resize(self.online_score, (s_size, s_size), interpolation=cv2.INTER_CUBIC)
            pscore = p.online_ratio * o_score + (1 - p.online_ratio) * pscore
        else:
            pass

        # get max
        r_max, c_max = np.unravel_index(pscore.argmax(), pscore.shape)

        # to real size
        pred_x1 = pred_x1[r_max, c_max]
        pred_y1 = pred_y1[r_max, c_max]
        pred_x2 = pred_x2[r_max, c_max]
        pred_y2 = pred_y2[r_max, c_max]

        pred_xs = (pred_x1 + pred_x2) / 2
        pred_ys = (pred_y1 + pred_y2) / 2
        pred_w = pred_x2 - pred_x1
        pred_h = pred_y2 - pred_y1

        diff_xs = pred_xs - p.instance_size // 2
        diff_ys = pred_ys - p.instance_size // 2

        diff_xs, diff_ys, pred_w, pred_h = diff_xs / scale_z, diff_ys / scale_z, pred_w / scale_z, pred_h / scale_z

        target_sz = target_sz / scale_z

        # size learning rate
        lr = penalty[r_max, c_max] * cls_score[r_max, c_max] * p.lr

        # size rate
        res_xs = target_pos[0] + diff_xs
        res_ys = target_pos[1] + diff_ys
        res_w = pred_w * lr + (1 - lr) * target_sz[0]
        res_h = pred_h * lr + (1 - lr) * target_sz[1]

        target_pos = np.array([res_xs, res_ys])
        target_sz = target_sz * (1 - lr) + lr * np.array([res_w, res_h])

        return target_pos, target_sz, cls_score[r_max, c_max]

    def track(self, state, im, online_score=None, gt=None):
        p = state['p']
        net = state['net']
        avg_chans = state['avg_chans']
        window = state['window']
        target_pos = state['target_pos']
        target_sz = state['target_sz']

        if online_score is not None:
            self.online_score = online_score.squeeze().cpu().data.numpy()
        else:
            self.online_score = None

        hc_z = target_sz[1] + p.context_amount * sum(target_sz)
        wc_z = target_sz[0] + p.context_amount * sum(target_sz)
        s_z = np.sqrt(wc_z * hc_z)
        scale_z = p.exemplar_size / s_z
        d_search = (p.instance_size - p.exemplar_size) / 2  # slightly different from rpn++
        pad = d_search / scale_z
        s_x = s_z + 2 * pad

        x_crop, _ = get_subwindow_tracking(im, target_pos, p.instance_size, python2round(s_x), avg_chans)
        x_crop = x_crop.unsqueeze(0)

        target_pos, target_sz, _ = self.update(net, x_crop.cuda(), target_pos, target_sz*scale_z, window, scale_z, p)

        target_pos[0] = max(0, min(state['im_w'], target_pos[0]))
        target_pos[1] = max(0, min(state['im_h'], target_pos[1]))
        target_sz[0] = max(10, min(state['im_w'], target_sz[0]))
        target_sz[1] = max(10, min(state['im_h'], target_sz[1]))
        state['target_pos'] = target_pos
        state['target_sz'] = target_sz
        state['p'] = p

        return state

    def grids(self, p):
        """
        each element of feature map on input search image
        :return: H*W*2 (position for each element)
        """
        sz = p.score_size

        # the real shift is -param['shifts']
        sz_x = sz // 2
        sz_y = sz // 2

        x, y = np.meshgrid(np.arange(0, sz) - np.floor(float(sz_x)),
                           np.arange(0, sz) - np.floor(float(sz_y)))

        self.grid_to_search_x = x * p.total_stride + p.instance_size // 2
        self.grid_to_search_y = y * p.total_stride + p.instance_size // 2

    def IOUgroup(self, pred_x1, pred_y1, pred_x2, pred_y2, gt_xyxy):
        # overlap

        x1, y1, x2, y2 = gt_xyxy

        xx1 = np.maximum(pred_x1, x1)  # 17*17
        yy1 = np.maximum(pred_y1, y1)
        xx2 = np.minimum(pred_x2, x2)
        yy2 = np.minimum(pred_y2, y2)

        ww = np.maximum(0, xx2 - xx1)
        hh = np.maximum(0, yy2 - yy1)

        area = (x2 - x1) * (y2 - y1)

        target_a = (pred_x2 - pred_x1) * (pred_y2 - pred_y1)

        inter = ww * hh
        overlap = inter / (area + target_a - inter)

        return overlap

    def change(self, r):
        return np.maximum(r, 1. / r)

    def sz(self, w, h):
        pad = (w + h) * 0.5
        sz2 = (w + pad) * (h + pad)
        return np.sqrt(sz2)

    def sz_wh(self, wh):
        pad = (wh[0] + wh[1]) * 0.5
        sz2 = (wh[0] + pad) * (wh[1] + pad)
        return np.sqrt(sz2)

class OceanConfig(object):
    penalty_k = 0.062
    window_influence = 0.38
    lr = 0.765
    windowing = 'cosine'
    exemplar_size = 127
    instance_size = 255
    total_stride = 8
    score_size = (instance_size - exemplar_size) // total_stride + 1 + 8  # for ++
    context_amount = 0.5
    ratio = 0.94

    def update(self, newparam=None):
        if newparam:
            for key, value in newparam.items():
                setattr(self, key, value)
            self.renew()

    def renew(self):
        self.score_size = (self.instance_size - self.exemplar_size) // self.total_stride + 1 + 8 # for ++


from django.shortcuts import render

# Create your views here.
def bics(request):

    colors=[[]]
    colors[0]=['rgb(0,0,0)','rgb(255,0,0)','rgb(0,0,255)','rgb(0,255,0)']
    former_shade=255
    new_shade=250
    for i in range(1,50):
        colors.append(['','','',''])
        colors[i][0]=colors[i-1][0].replace(str(255-int(former_shade)),str(255-int(new_shade)))
        for j in range(1,4):
            colors[i][j]=colors[i-1][j].replace(str(former_shade),str(new_shade))
        former_shade-=5
        new_shade-=5
    context={'colors':colors}

    return render(request,'template_bics.html',context)


# ccc02s1

'''
Sample Input
Copy

1
2
3
4
3

Sample Output
Copy

# of PINK is 0 # of GREEN is 0 # of RED is 1 # of ORANGE is 0
# of PINK is 1 # of GREEN is 1 # of RED is 0 # of ORANGE is 0
# of PINK is 3 # of GREEN is 0 # of RED is 0 # of ORANGE is 0
Total combinations is 3.
Minimum number of tickets to print is 1.
'''

pink, green, yellow, orange, i = (int(input()) for _ in range(5))
sols = []
for p in range(1 + i // pink):
    for g in range(1 + i // green):
        for y in range(1 + i // yellow):
            for o in range(1 + i // orange):
                if (p * pink + g * green + y * yellow + o * orange == i):
                    sols.append((p, g, y, o))
for sol in sols: print("# of PINK is {} # of GREEN is {} # of RED is {} # of ORANGE is {}".format(sol[0], sol[1], sol[2], sol[3]))
print("Total combinations is {}.".format(len(sols)))
print("Minimum number of tickets to print is {}.".format(min(map(sum, sols))))


# -------------------------------------------------------------------
# -- Object Oriented Programming => Class Methods & Static Methods --
# -------------------------------------------------------------------
# Class Methods:
# - Marked With @classmethod Decorator To Flag It As Class Method
# - It Take Cls Parameter Not Self To Point To The Class not The Instance
# - It Doesn't Require Creation of a Class Instance
# - Used When You Want To Do Something With The Class Itself
# Static Methods:
# - It Takes No Parameters
# - Its Bound To The Class Not Instance
# - Used When Doing Something Doesnt Have Access To Object Or Class But Related To Class
# -----------------------------------------------------------

class Member:
    not_allowed_names = ["Hell", "Shit", "Damn"]
    users_num = 0

    @classmethod
    def show_users_count(cls):
        print(f"We Have {cls.users_num} Users In Our System.")

    @staticmethod
    def say_hello():
        print("Hello From Static Method")

    def __init__(self, first_name, middle_name, last_name, gender):
        self.fname = first_name
        self.mname = middle_name
        self.lname = last_name
        self.gender = gender
        Member.users_num += 1  # Member.users_num = Member.users_num + 1

    def full_name(self):
        if self.fname in Member.not_allowed_names:
            raise ValueError("Name Not Allowed")
        else:
            return f"{self.fname} {self.mname} {self.lname}"

    def name_with_title(self):
        if self.gender == "Male":
            return f"Hello Mr {self.fname}"
        elif self.gender == "Female":
            return f"Hello Miss {self.fname}"
        else:
            return f"Hello {self.fname}"

    def get_all_info(self):
        return f"{self.name_with_title()}, Your Full Name Is: {self.full_name()}"

    def delete_user(self):
        Member.users_num -= 1  # Member.users_num = Member.users_num -1
        return f"User {self.fname} Is Deleted."

print(Member.users_num)

member_one = Member("Osama", "Mohamed", "Elsayed", "Male")
member_two = Member("Ahmed", "Ali", "Mahmoud", "Male")
member_three = Member("Mona", "Ali", "Mahmoud", "Female")
member_four = Member("Shit", "Hell", "Metal", "Damn")

print(Member.users_num)
print(member_four.delete_user())
print(Member.users_num)

print("=" * 50)  # Separator

Member.show_users_count()

print("=" * 50)  # Separator

print(member_one.full_name())
print(Member.full_name(member_one))

Member.say_hello()

Lessons/Week 14/108 - OOP - Part 06 - Class Methods & Static Methods.py


#!/usr/bin/python3
"""module for canUnlockAll function"""

def canUnlockAll(boxes):
    """
    You have n number of locked boxes in front of you.
    Each box is numbered sequentially from 0 to n - 1 and
    each box may contain keys to the other boxes.
    A key with the same number as a box opens that box
    You can assume all keys will be positive integers
    There can be keys that do not have boxes
    The first box boxes[0] is unlocked

    Write a method that determines if all the boxes can be opened.

    boxes - a list of lists

    Return True if all boxes can be opened, else return False
    """
    keys = [0]
    opened_boxes = 0
    while opened_boxes < len(boxes):
        if keys[0] < len(boxes) and boxes[keys[0]] != [-1]:
            keys += [x for x in boxes[keys[0]] if x <
                     len(boxes) and boxes[x] != [-1]]
            boxes[keys[0]] = [-1]
            opened_boxes += 1
        keys.remove(keys[0])
        if all(box == [-1] for box in boxes):
            return True
        if keys == []:
            return False
    return True


from flask import request, jsonify
from flask.views import MethodView
from flask.ext.login import current_user

from server.models.food import FoodEntry
from server.api.sessionauth import session_auth_required

from server import app, db, login_manager

class MenuAPI(MethodView):
    def get(self):
        menu_items = FoodEntry.query.filter_by(active=True).all()

        item_list = []

        for item in menu_items:
            obj = {
                'name': item.name,
                'active': item.active,
                'price': item.price,
                'restaurant': item.restaurant.name,
                'details': item.details,
                'ingredients': item.ingredients,
                'nutrition': item.nutrition,
                'imageurl_icon': item.imageurl_icon,
                'imageurl_full': item.imageurl_full,
            }

            obj['tags'] = [tag.name for tag in item.tags]

            item_list.append(obj)

        response_json = {'items': item_list}

        return jsonify(**response_json)

menu_view = MenuAPI.as_view('menu_api')
app.add_url_rule('/api/menu', view_func=menu_view, methods=['GET'])


import cv2 as cv

img1 = cv.imread(r'media/images.jpg',
                 cv.IMREAD_GRAYSCALE) #convert image to the single channel grayscale image
img2 = cv.imread(r'media/47428-7-smiley-picture-free-photo-png.png',
                 cv.IMREAD_REDUCED_COLOR_2) #convert image to the 3 channel BGR color image and the image size reduced 1/2
img3 = cv.imread(r'media/unknown.png',
                 cv.IMREAD_ANYCOLOR) #the image is read in any possible color format

cv.namedWindow('window 1',
               cv.WINDOW_NORMAL) #the user can resize the window
cv.namedWindow('window 2',
               cv.WINDOW_AUTOSIZE) #the user cannot resize the window, the size is constrainted by the image displayed.
cv.namedWindow('window 3',
               cv.WINDOW_GUI_EXPANDED) #status bar and tool bar

cv.imshow('picture 1', img1)
cv.imshow('picture 2', img2)
cv.imshow('picture 3', img3)

cv.waitKey(0)

cv.destroyAllWindows()


import boto3
from boto3.dynamodb.conditions import Key

dynamodb = boto3.resource('dynamodb')

def create_table_with_gsi():
    table = dynamodb.create_table(
        TableName='Employees',
        KeySchema=[
            {
                'AttributeName': 'emp_id',
                'KeyType': 'HASH'
            },
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'emp_id',
                'AttributeType': 'N'
            },
            {
                'AttributeName': 'email',
                'AttributeType': 'S'
            },

        ],
        GlobalSecondaryIndexes=[
            {
                'IndexName': 'email',
                'KeySchema': [
                    {
                        'AttributeName': 'email',
                        'KeyType': 'HASH'
                    },
                ],
                'Projection': {
                    'ProjectionType': 'ALL'
                },
                'ProvisionedThroughput' :{
                    'ReadCapacityUnits': 1,
                    'WriteCapacityUnits': 1,
                }
            }
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 1,
            'WriteCapacityUnits': 1,
        }
    )

    print("Table status:", table.table_status)

def create_employee():
    user = {
        'emp_id': 1,
        'first_name': 'Jon',
        'last_name': 'Doe',
        'email': 'jdoe@test.com'
    }

    dynamodb = boto3.resource('dynamodb')

    table = dynamodb.Table('Employees')

    table.put_item(Item=user)

def query_data_with_gsi():
    table = dynamodb.Table('Employees')

    response = table.query(
        IndexName='email',
        KeyConditionExpression=Key('email').eq('jdoe@test.com')
    )

    print(response['Items'][0])
    #{'last_name': 'Doe', 'email': 'jdoe@test.com', 'first_name': 'Jon', 'emp_id': Decimal('1')}

def create_table_with_lsi():
    table = dynamodb.create_table(
        TableName='Posts',
        KeySchema=[
            {
                'AttributeName': 'user_name',
                'KeyType': 'HASH'
            },
            {
                'AttributeName': 'title',
                'KeyType': 'RANGE'
            }
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'title',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'user_name',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'subject',
                'AttributeType': 'S'
            },

        ],
        LocalSecondaryIndexes=[
            {
                'IndexName': 'user_name_subject',
                'KeySchema': [
                    {
                        'AttributeName': 'user_name',
                        'KeyType': 'HASH'
                    },
                    {
                        'AttributeName': 'subject',
                        'KeyType': 'RANGE'
                    },
                ],
                'Projection': {
                    'ProjectionType': 'ALL'
                },
            }
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 1,
            'WriteCapacityUnits': 1,
        }
    )

    print("Table status:", table.table_status)

def create_posts():
    post1 = {
        'title': "My favorite hiking spots",
        'user_name': 'jon_doe',
        'subject': 'hiking'
    }

    post2 = {
        'title': "My favorite recipes",
        'user_name': 'jon_doe',
        'subject': 'cooking'
    }

    post3 = {
        'title': "I love hiking!",
        'user_name': 'jane_doe',
        'subject': 'hiking'
    }

    dynamodb = boto3.resource('dynamodb')

    table = dynamodb.Table('Posts')

    table.put_item(Item=post1)
    table.put_item(Item=post2)
    table.put_item(Item=post3)


class Solution:
    def removeDigit(self, number: str, digit: str) -> str:  # 38.93% 16.09%
        idx = None
        for i, char in enumerate(number):
            if char == digit:
                idx = i
            elif idx is not None and idx == i-1 and char > digit:
                return number[:idx] + number[idx+1:]
        return number[:idx] + number[idx+1:]

    def removeDigit_best_speed(self, n: str, d: str) -> str:
        res = ''
        for i in range(len(n)):
            if n[i] == d:
                if res == '':
                    res = n[:i] + n[i+1:]
                elif res < n[:i] + n[i+1:]:
                    res = n[:i] + n[i+1:]
        return res

    def removeDigit_best_memory(self, number: str, digit: str) -> str:
        largest = ''
        for i in range(len(number)):
            if number[i] == digit and number[:i] + number[i+1:] > largest:
                largest = number[:i] + number[i+1:]
        return largest

2200 - 2299/2259 - Remove Digit From Number to Maximize Result/2259 - Remove Digit From Number to Maximize Result.py


# import necessary tools
import time
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
# importing the naive bayes model class MultinomialNB (NaiveBayes), which works well with count_vectorizers as it expects integer inputs
#from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
#from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
# metric model to evaluate the model performance
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfTransformer

start_time = time.time()

# load data into DataFrame
comments = pd.read_csv('attack_annotated_comments.tsv', sep='\t', index_col=0)
annotations = pd.read_csv('attack_annotations.tsv', sep='\t')

# print the # of unique rev_id
print('There are', len(annotations['rev_id'].unique()), 'unique rev_id')

# labels a comment as an attack if the majority of annotators did so
labels = annotations.groupby('rev_id')['attack'].mean() > 0.5

# insert labels in comments
comments['attack'] = labels

# Parsing: remove newline and tab tokens
comments['comment'] = comments['comment'].apply(lambda x: x.replace("NEWLINE_TOKEN", " "))
comments['comment'] = comments['comment'].apply(lambda x: x.replace("TAB_TOKEN", " "))

print('This is your DataFrame:\n')
print(comments.head())
print('These are the head of the comments classified as an attack\n')
print(comments.query('attack')['comment'].head())

# create y which is the outcome label the model has to learn
y = comments['attack']

'''
split the dataframe into training and testing data
split the features X (that are the comment column, comments['comment'])
and the label y (attack, comments['attack']) based on a given test size such as 0.33 (33%)
random state, so we can have repeatable results when we run the code again
the function will take 33% of rows to be marked as test data and move them from the training data.
the test data is later used to see how the model has learned

The resulting data from train_test_split() are:
training data as X_train, training labels as y_train,
testing data as X_test and testing labels as y_test
'''

X_train, X_test, y_train, y_test = train_test_split(comments['comment'], y, test_size=0.33, random_state=53)

# create count vectorizer that turn the text into a bag-of-words vectors
# each tokens acts as a feature for the machine learning classification problem

max_features = (5000, 10000, 50000)
C = (0.0001, 0.001, 0.01, 0.1, 1, 10)

# Specify the hyperparameter space
# char-gram, no tfidf

param_grid = {
    'vec__max_features': max_features,
    'vec__ngram_range': ((1, 5),),
    'vec__analyzer': ('char',),
    'tfidf__sublinear_tf': (True, False),
    'tfidf__norm': (None, 'l2'),
    'clf__C': C,
}

# Setup the pipeline
steps = [('vec', CountVectorizer()),
         ('tfidf', TfidfTransformer()),
         ('clf', LogisticRegression())]

pipeline = Pipeline(steps)

load_time = time.time()

# Instantiate the GridSearchCV object: cv
cv = GridSearchCV(pipeline, param_grid, cv=12)

# Fit to the training set
cv.fit(X_train, y_train)

build_time = time.time()

# Predict the labels of the test set: y_pred
y_pred = cv.predict(X_test)

# Compute and print metrics
print("Accuracy: {}".format(cv.score(X_test, y_test)))
print("Tuned Model Parameters: {}".format(cv.best_params_))
print(metrics.classification_report(y_test, y_pred))
eval_time = time.time()

print("Times: %0.3f sec loading, %0.3f sec building, %0.3f sec evaluation" % (load_time - start_time, build_time - load_time, eval_time - build_time,))
print("Total time: %0.3f seconds" % (eval_time - start_time))


import numpy as np,pandas as pd,datetime as dt

import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

from flask import Flask, jsonify

#################################################
# Database Setup
#################################################
engine = create_engine("sqlite:///hawaii.sqlite")

# reflect an existing database into a new model
Base = automap_base()
# reflect the tables
Base.prepare(autoload_with=engine,reflect=True)

# Save reference to the table
Base.classes.keys()

# Save reference to the table
Measurement = Base.classes.measurement

Station = Base.classes.station
# Create our session (link) from Python to the DB
session = Session(engine)
#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################

@app.route("/")
def welcome():
    """List all available api routes."""
    return (
        "Available Routes:
/api/v1.0/precipitation:
api/v1.0/stations:
/api/v1.0/tobs:
"
        )

#
@app.route("/api/v1.0/precipitation")
def precipitation():
    # Create our session (link) from Python to the DB
    session = Session(engine)
    recent_date = session.query(Measurement.date).order_by(Measurement.date.desc()).first().date
    last_year = dt.datetime.strptime(recent_date, '%Y-%m-%d') - dt.timedelta(days=365)

    # Perform a query to retrieve the data and precipitation scores

    prcp_data = session.query(Measurement.date, Measurement.prcp).filter(Measurement.date >= last_year).all()

    # Save the query results as a Pandas DataFrame and set the index to the date column
    prcp_df = pd.DataFrame(prcp_data,columns = ["Date","Precipitaion"]).sort_values(by="Date")
    prcp_df.set_index(prcp_df['Date'], inplace=True)
    # Sort the dataframe by date
    session.close()

    """Precipitation analysis (i.e. retrieve only the last 12 months of data)."""
    # Query precipitation data
    result = {}
    for index, row in prcp_df.iterrows():
        #result[index] = row.to_json()
        result[index] = dict(row)
    return jsonify(result)

#Return a JSON list of stations from the dataset

@app.route('/api/v1.0/stations')
def stations():
    session = Session(engine)
    station = session.query(Station.station).all()

    d = {k[0]:'' for k in station}
    session.close()

    return jsonify(d)

#Query the dates and temperature observations of the most-active station for the previous year of data.
#Return a JSON list of temperature observations for the previous year.

@app.route('/api/v1.0/tobs')
def most_active():
    session = Session(engine)
    most_active = session.query(Measurement.station, func.count(Measurement.station)).\
              group_by(Measurement.station).\
              order_by(func.count(Measurement.station).desc()).all()
    print(most_active)
    session.close()

# Select the station with the highest number of temperature observations
    most_active_station = most_active[0][0]

# Query the most active station's temperature data for the previous year
    temps_last_year = session.query(Measurement.tobs).\
                      filter(Measurement.station == most_active_station).\
                      filter(Measurement.date >= '2016-08-23').all()

# Convert the query results to a list
    temps_list = list(np.ravel(temps_last_year))

    # Return the JSON representation of the list
    return jsonify(temps_list)

# 5
# /api/v1.0/<start> and /api/v1.0/<start>/<end>
# Return a JSON list of the minimum temperature, the average temperature, and the maximum temperature for a specified start or start-end range.
# For a specified start, calculate TMIN, TAVG, and TMAX for all the dates greater than or equal to the start date.
# For a specified start date and end date, calculate TMIN, TAVG, and TMAX for the dates from the start date to the end date, inclusive.

@app.route('/api/v1.0/<start>')
@app.route('/api/v1.0/<start>/<end>')
def min_max_avg(start, end=None):
    session = Session(engine)
    if end:
        results = session.query(func.min(tobs), func.avg(tobs), func.max(tobs)).\
                  filter(Measurement.date >= start).filter(Measurement.date <= end).all()
    else:
        results = session.query(func.min(tobs), func.avg(tobs), func.max(tobs)).\
                  filter(Measurement.date >= start).all()

    session.close()

    all_temps = []
    for min_temp, avg_temp, max_temp in results:
        temp_dict = {}
        temp_dict['min_temp'] = min_temp
        temp_dict['avg_temp'] = avg_temp
        temp_dict['max_temp'] = max_temp
        all_temps.append(temp_dict)

    return jsonify(all_temps)

if __name__ == '__main__':
    app.run(debug=True)


from turtle import *

colors = ["red","blue","brown","yellow","grey"]

# for i in range(5):
#     pencolor(colors[i])
#     for _ in range(i+3):
#         forward(50)
#         left(360/(i+3))

for i in range(4):
    fillcolor(colors[i])
    pencolor(colors[i])
    begin_fill()
    for _ in range(4):
        forward(100)
        left(90)
    forward(50)
    end_fill()
fillcolor(colors[4])
pencolor(colors[4])
begin_fill()
for _ in range(2):
    forward(50)
    left(90)
    forward(100)
    left(90)
end_fill()

mainloop()


# _*_ coding: utf-8 _*_
# @Time : 2018/9/15 下午2:31
# @Author :Daniel
# @File : mnist_test.py
# @Software: PyCharm

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist=input_data.read_data_sets("mnist/",one_hot=True)

inputNode=784
outputNode=10

layer1_Node=500
batch=10

x=tf.placeholder(tf.float32,[None,inputNode],name='x_input')

y_=tf.placeholder(tf.float32,[None,outputNode],name='y_input')

weights1=tf.Variable(tf.truncated_normal([inputNode,layer1_Node],stddev=0.1))
biases1=tf.Variable(tf.constant(0.1,shape=[layer1_Node]))

weights2=tf.Variable(tf.truncated_normal([layer1_Node,outputNode],stddev=0.1))
biases2=tf.Variable(tf.constant(0.1,shape=[outputNode]))

layer1=tf.nn.relu(tf.matmul(x,weights1)+biases1)
y=tf.nn.relu(tf.matmul(layer1,weights2)+biases2)

# globals_step=tf.Variable(0,trainable=False)

cross_entropy=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
cross_entropy_mean=tf.reduce_mean(cross_entropy)

train_step=tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy_mean)

correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))

with tf.Session() as sess:
    tf.global_variables_initializer().run()
    validate_feed={x:mnist.train.images,y_:mnist.train.labels}
    test_feed={x:mnist.test.images,y_:mnist.test.labels}

    Step=5000
    for i in range(Step):
        if i%10==0:
            validate_acc=sess.run(accuracy,feed_dict=test_feed)
            print("after %d training steps(s), validation accuracy is %g"%(i,validate_acc))
        xs,ys=mnist.train.next_batch(batch)
        # sess.run(cross_entropy_mean,feed_dict=validate_feed)
        # print("运行{}次".format(i))
        sess.run(train_step,feed_dict={x:xs,y_:ys})
        # print(cross_enftropy_mean)
        print("正运行{}次".format(i))


import requests
from bs4 import BeautifulSoup
import urllib
import json
import re

__author__ = 'Sushim'

class Phone:
    name = ''
    memory = ''
    camera = ''
    battery = ''
    ram = ''
    price = ''
    image = ''

    def print(self):
        """
        Print function for Phone class
        """
        print('Name:', self.name)
        print('Camera:', self.camera)
        print('Memory:', self.memory)
        print('Ram:', self.ram)
        print('Price:', self.price)
        print('Image:', self.image)

    def __init__(self):
        return

def getLinkstoBrands(url):
    """
    function gets the links to all the brands
    from the marker page of the website
    :param url: string,
    :return: dictionary of brandnames, values as links to their all display section
    """
    brandUrls = {}
    try:
        print("Maker link being crawled  : ", url)
        request = requests.get(url)
        if request.status_code == 200:
            sourceCode = BeautifulSoup(request.text, "html.parser")
            for td in sourceCode.findAll('td'):
                link = td.find('a', href=True)
                title = td.get_text()
                url = processUrl(link['href'])
                if title not in brandUrls.keys():
                    brandUrls[title] = url
                    print(title, ' ', url)
        else:
            print('no table or row found ')
    except requests.HTTPError as e:
        print('Unable to open url', e)
    return brandUrls

def getLinksToPhonesPerBrands(url):
    """
    function to return all phone's urls for a particular brand
    :param url: url of brand
    :return: dictionary of phones, links to phones as values
    """
    urls = {}
    print("brand link being scrapped : ", url)
    try:
        request = requests.get(url)
        if request.status_code == 200:
            sourceCode = BeautifulSoup(request.content, "html.parser")
            li = sourceCode.select('#review-body div > ul > li > a')
            for link in li:
                title = link.get_text()
                url = processUrl(link['href'])
                if title not in urls.keys():
                    urls[title] = url
                    print(title, ' ', url)
        else:
            print('no table or row found ')
    except requests.HTTPError as e:
        print('Unable to open url', e)
    return urls

def getPhoneStats(url):
    """
    function to extract phone details from each phone page
    :param url: string, url of the phone's page.
    :return: Phone type object, with mem, camera, ram and price details
    """
    p = Phone()
    try:
        request = requests.get(url)
        if request.status_code == 200:
            soup = BeautifulSoup(request.text, "html.parser")
            p.ram = get_ram(soup)
            p.memory = get_memory(soup)
            p.image = get_image(soup)
            for table in soup.findAll("table"):
                header = table.th.get_text()
                if header == 'Camera':
                    p.camera = get_camera(table)
                if header == 'Battery':
                    p.battery = get_battery(table)
                if header == 'Misc':
                    p.price = get_price(table)
        else:
            print('unable to connect ')
    except requests.HTTPError as e:
        print('Unable to open url', e)
    return p

def processUrl(url):
    """
    function to create complete processable url
    :param url: string, url
    :return: string, finished url
    """
    domain = 'http://www.gsmarena.com/'
    if domain not in url:
        url = urllib.parse.urljoin(domain, url)
    return url

def get_ram(soup):
    """
    function to extract ram details from the page HTML text
    :param soup: BeautifulSoup object, (contains page text)
    :return: ram, integer.
    """
    ram = soup.find("strong", {"class": "accent accent-expansion"})
    if len(ram):
        ram = re.findall(r'\d+\.*\d*', ram.get_text())
        if len(ram):
            ram = float(ram[0])
        else:
            ram = 0
    else:
        ram = 0
    return ram

def get_memory(soup):
    """
    function to extract memory details from the page HTML text
    :param soup: BeautifulSoup object, (contains page text)
    :return: memory, integer.
    """
    memory = soup.find_all("span", {"class": "specs-brief-accent"})
    if len(memory):
        memory = re.findall(r'\d+', memory[3].get_text())
        if len(memory):
            memory = int(memory[0])
        else:
            memory = 0
    else:
        memory = 0
    return memory

def get_camera(table):
    """
    function to extract camera details from the page HTML text
    :param table: table object from BeautifulSoup parse, (contains page text)
    :return: camera, integer.
    """
    camera = table.findAll("tr")[0].findAll("td")[1]
    if len(camera):
        camera = re.findall(r'\d+', camera.get_text())
        if len(camera):
            camera = int(camera[0])
        else:
            camera = 0
    else:
        camera = 0
    return camera

def get_battery(table):
    """
    function to extract battery details from the page HTML text
    :param table: table object from BeautifulSoup parse, (contains page text)
    :return: battery, integer.
    """

    battery = table.findAll("tr")[0].findAll("td")[1]

    if battery is not None:
        battery = re.search(r'\d+', battery.get_text())
        if battery is not None:
            battery = int(battery.group())
        else:
            battery = 0
    else:
        battery = 0
    return battery

def get_price(table):
    """
    function to extract price details from the page HTML text
    :param table: table object from BeautifulSoup parse, (contains page text)
    :return: price, integer.
    """
    price = table.find("span", {"class": "price"})
    if price is not None:
        price = re.search(r'\d+', price.get_text())
        if price is not None:
            price = int(price.group())
        else:
            price = 0
    else:
        price = 0
    return price

def get_image(soup):
    """
    function returns the image url of the phone
    :param soup: BeautifulSoup object
    :return: url of image for thumbnail
    """
    image = soup.find("div", {"class": "specs-photo-main"}).find('img')['src']
    return image

def writeToFile(jsonList):
    """
    function to write JSON objects to file.
    :param p: List of Json formatted Phone object
    """
    file = open("data.json", "w")
    file.write(json.dumps(jsonList, sort_keys=True, indent=4, separators=(',', ': ')))
    print("File written")

def toJson(p):
    """
    function to convert Phone object to JSON
    :param p:
    :return: json format data
    """
    data = {p.name: {'Memory': p.memory, 'Camera': p.camera, 'Battery': p.battery, 'Ram': p.ram, 'Price': p.price,
                     'Image url': p.image}}
    return data

if __name__ == '__main__':
    linksToBrands = getLinkstoBrands('http://www.gsmarena.com/makers.php3')
    jsonList = []
    for link in linksToBrands:
        phoneUrls = getLinksToPhonesPerBrands(linksToBrands[link])
        for links in phoneUrls:
            print(" Phone being scraped : ", links, ' : ', phoneUrls[links])
            p = getPhoneStats(phoneUrls[links])
            p.name = links
            jsonList.append(toJson(p))
    print("Writing to file")
    writeToFile(jsonList)


import threading
import sys
import time
import datetime
import json
import numpy as np

from pslab.instrument.digital import MODES

MODES = {v: k for k, v in MODES.items()}

class LogicAnalyser:
    def __init__(self, I, file_write):
        self.file_write = file_write
        self.capture_time = 0.1
        self.device = I
        self.la_read_thread = None
        self.is_reading = False
        self.channel_mode = 3
        self.number_of_channels = 4
        self.trigger1_type = 1
        self.trigger2_type = 1
        self.trigger3_type = 1
        self.trigger4_type = 1

    def set_config(self, number_of_channels, trigger1_type, trigger2_type, trigger3_type, trigger4_type, capture_time):
        self.number_of_channels = int(number_of_channels)
        self.trigger1_type = int(trigger1_type)
        self.trigger2_type = int(trigger2_type)
        self.trigger3_type = int(trigger3_type)
        self.trigger4_type = int(trigger4_type)
        self.capture_time = int(capture_time)

    def get_config(self):
        output = {'type': 'GET_CONFIG_LA',
                  'numberOfChannels': self.number_of_channels,
                  'trigger1Type': self.trigger1_type,
                  'trigger2Type': self.trigger2_type,
                  'trigger3Type': self.trigger3_type,
                  'trigger4Type': self.trigger4_type,
                  'captureTime': self.capture_time
                  }
        print(json.dumps(output))
        sys.stdout.flush()

    def start_read(self):
        self.la_read_thead = threading.Thread(
            target=self.capture,
            name='la')
        self.la_read_thead.start()

    def stop_read(self):
        if self.is_reading:
            self.is_reading = False
            self.device.logic_analyzer.stop()
            self.la_read_thead.join()

    def capture(self):
        self.is_reading = True
        self.device.logic_analyzer.capture(
                self.number_of_channels,
                modes=[
                        MODES[self.trigger1_type],
                        MODES[self.trigger2_type],
                        MODES[self.trigger3_type],
                        MODES[self.trigger4_type],
                ],
                block=False
        )
        time.sleep(self.capture_time / 1e3)
        self.device.logic_analyzer.stop()
        timestamps = self.device.logic_analyzer.fetch_data()
        xy = 8 * [np.array([])]
        xy[: self.number_of_channels * 2] = self.device.logic_analyzer.get_xy(timestamps)
        x1, y1, x2, y2, x3, y3, x4, y4 = xy

        output = {
            'type': 'START_LA',
            'time1': x1[1:].tolist(),
            'voltage1': (y1[1:] + 0.).tolist(),
            'time2': x2[1:].tolist(),
            'voltage2': (y2[1:] + 2.).tolist(),
            'time3': x3[1:].tolist(),
            'voltage3': (y3[1:] + 4.).tolist(),
            'time4': x4[1:].tolist(),
            'voltage4': (y4[1:] + 6.).tolist(),
            'numberOfChannels': self.number_of_channels,
        }
        if self.is_reading:
            print(json.dumps(output))
            sys.stdout.flush()
        self.is_reading = False


import numpy as np
import pandas as pd
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

def transform_data(xtrain, ytrain, xtest, ytest, n_components=None, use_pca=False):
    """
    Apply feature scaling, dimensionality reduction to the data. Return the standardized
    and low-dimensional train and test sets together with the scaler object for the
    target values.
    Arguments:
        xtrain: size=(ntrain, p),
            training input
        ytrain: size=(ntrain, ?),
            training truth, ? depends on what we train against (mulitple objective)
        xtest: size=(ntest, p),
            testing input
        ytest: size=(ntest, ?),
            testing truth
        n_components: int,
            number of principal components used if use_pca=True
        use_pca: bool,
            if true use principal component analysis for dimensionality reduction

    Returns:
        xtrain_scaled, ytrain_scaled, xtest_scaled, ytest_scaled, yscaler
    """

    xscaler = StandardScaler()
    xtrain_scaled = xscaler.fit_transform(xtrain)
    xtest_scaled = xscaler.transform(xtest)
    yscaler = StandardScaler()
    ytrain_scaled = yscaler.fit_transform(ytrain)
    ytest_scaled = yscaler.transform(ytest)

    if use_pca:
        pca = PCA(n_components)
        xtrain_scaled = pca.fit_transform(xtrain)
        print("Fraction of variance retained is: ", np.sum(pca.explained_variance_ratio_))
        xtest_scaled = pca.transform(xtest)
    return xtrain_scaled, xtest_scaled, ytrain_scaled, ytest_scaled, yscaler

def load_data(data_path, target_path, target):
    df_target = pd.read_csv(target_path)
    df = pd.read_csv(data_path)
    target_vector = []
    for i, name1 in enumerate(df["id"]):
        for j, name2 in enumerate(df_target["id"]):
            if name1 == name2:
                target_vector.append(df_target[target][j])
    removals = ["id"]
    df = df.drop(columns=removals)
    df = df.dropna(axis=1)
    return df, target_vector

def load_data_cv(data_path, target_path, target):
    df_target = pd.read_csv(target_path)
    df = pd.read_csv(data_path)
    target_vector = []
    for i, name1 in enumerate(df["id"]):
        for j, name2 in enumerate(df_target["id"]):
            if name1 == name2:
                target_vector.append(df_target[target][j])
    removals = ["id"]
    df = df.drop(columns=removals)
    df = df.dropna(axis=1)
    df["target"] = target_vector
    return df

def scale_features(xtrain, xtest):
    xscaler = StandardScaler()
    xtrain_scaled = xscaler.fit_transform(xtrain)
    xtest_scaled = xscaler.transform(xtest)
    return xtrain_scaled, xtest_scaled, xscaler


#!/usr/bin/env python3

import requests
import json
import time

import asyncio

from status_res import StatusRes

class Connector:

    SERVER_URL = 'https://32df8992-903f-49f5-9b4d-4c10e1bfeb19.mock.pstmn.io'
    PATH = '/status2'
    PATH2='/started'
    # /status => 'running'
    # /status2 => 'completed'
    # /start => 'started'

    def __init__(self):
        self.api1_status = None
        self.api2_status = None
        self.api3_status = None

    async def call_api1(self):
        print('making network call for request:  api1')
        # simulate network delay
        time.sleep(5)
        r = requests.get(self.SERVER_URL+self.PATH)
        j = r.json()
        self.api1_status = StatusRes(**j)
        print('got network response for request: api1')
        return self.api1_status.status

    async def call_api2(self):
        print('making network call for request:  api2')
        # simulate network delay
        time.sleep(3)
        r = requests.get(self.SERVER_URL+self.PATH)
        j = r.json()
        self.api2_status = StatusRes(**j)
        print('got network response for request: api2')
        return self.api2_status.status

    async def call_api3(self):
        print('making network call for request:  api3')
        # simulate network delay
        time.sleep(1)
        r = requests.get(self.SERVER_URL+self.PATH)
        j = r.json()
        self.api3_status = StatusRes(**j)
        print('got network response for request: api3')
        return self.api3_status.status

    def is_ready_to_update(self):
         return self.api1_status.is_completed() and self.api2_status.is_completed() and self.api3_status.is_completed()

    async def update_api1(self):
        print('making network call to update:  api1')
        time.sleep(1)

    async def update_api2(self):
        print('making network call to update:  api2')
        time.sleep(3)

    async def update_api3(self):
        print('making network call to update:  api3')
        time.sleep(5)


import logging

from .attr_dict import AttrDict

class Inputs(dict):
    """
    Inputs implementation (just dict with some utility methods)
    """

    def __init__(self, *args, sources=None, config=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.exposed = {}  # mapping name to source
        self.sources = {}
        self.config = AttrDict(config)
        if not sources:
            return
        for source in sources:
            self.register(source.__class__.__name__, source)

    def __str__(self):
        return f"<yapp inputs {len(self)}>"

    def __repr__(self):
        keys = set(self.keys()) | set(self.keys())
        return f"<yapp inputs {len(self)} {keys}>"

    def __getitem__(self, key):
        try:
            logging.debug('Using input "%s"', key)
            # if it's an exposed resource from an adapter return it
            if key in self.exposed:
                source, name = self.exposed[key]
                return self.sources[source][name]
            return super().__getitem__(key)
        except KeyError as error:
            # allow accessing config from jobs
            # not sure if this will remain or not (for sure not here)
            # I didn't add self['config'] in __init__ to keep the right length
            if key == "config":
                return self.config

            logging.debug('%s Trying to load missing input "%s"', self.__repr__(), key)
            raise KeyError(f'Trying to load missing input "{key}"') from error

    def __setitem__(self, key, value):
        if key in self.exposed:
            raise ValueError("Cannot assign to exposed input from adapter")
        super().__setitem__(key, value)

    def update(self, other, **kwargs):
        if isinstance(other, Inputs):
            self.sources.update(other.sources)
            self.exposed.update(other.exposed)
        super().update(other, **kwargs)

    def __or__(self, _):
        raise NotImplementedError

    def register(self, name: str, adapter):
        """
        New input adapter (just a new Item)
        """
        self.sources[name] = adapter
        logging.info('Registered new input source: "%s"', name)

    def expose(self, source, internal_name, name):
        """
        Expose input attribute using another name
        """
        # add an empty value instead of overriding all special methods like __len__, __contains__
        self[name] = None
        self.exposed[name] = (source, internal_name)
        logging.info('Exposed "%s" from "%s" as "%s"', internal_name, source, name)


# cleaning empty cells

import pandas as pd

df = pd.read_csv("C:\\Users\\hp pavilion\\Documents\\Python\\Machine learning\\Python_libraries\\Pandas\\resources\\Titanic_Uncleaned\\titanic_train.csv")

# sample view
# print(df.head(5))

# check for the precense of Empty Cells
print(df.isnull())  # sample empty cells :- col: Cabin- 0,2,4,...,886,888,890 ; col: age- 888

# method 1 - droping the rows that contain empty cells
cleaned_df = df.dropna()  # without affecting the original dataframe
# print(cleaned_df.isnull().to_string())  # all cells will have False value

# df.dropna(inplace=True)  # by affecting the original dataframe
# print(df.isnull())  # all cell must have False value

# method 2 - replacing null values
# replaced_df1 = df.fillna(value="unspecified")
# replaced_df2 = df.fillna(value="unspecified", inplace=True)  # replaces the value and returns Nane
# print(replaced_df1)
# print(replaced_df2)  # Nane

# replacing empty cells of the specified column
# df['Age'] = df['Age'].fillna('AGE')
# df['Cabin'] = df['Cabin'].fillna('CABIN')

# replacing empty cells with mean, median and mode
# target columns:- Age, Cabin
mean = df['Age'].mean()   # mean = 29.69911764705882
median = df['Fare'].median()  # median = 14.4542
mode = df['Cabin'].mode()[0]   # mode = B96 B98

df['Age'] = df['Age'].fillna(value=mean)  # notice row 888 - the value must be : 29.699118
df['Cabin'] = df['Cabin'].fillna(value=mode)  # notice row 0,2,4,...,886,888,890 - value : B96 B98
df['Fare'] = df['Fare'].fillna(value=median)  # notice row 887 - value : 14.4542
print(df.to_string())


import pytest
from freezegun import freeze_time
from sqlalchemy.exc import IntegrityError

from app import encryption
from app.constants import (
    EMAIL_TYPE,
    MOBILE_TYPE,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_FAILED,
    NOTIFICATION_PENDING,
    NOTIFICATION_SENDING,
    NOTIFICATION_STATUS_LETTER_ACCEPTED,
    NOTIFICATION_STATUS_LETTER_RECEIVED,
    NOTIFICATION_STATUS_TYPES_FAILED,
    NOTIFICATION_TECHNICAL_FAILURE,
    PRECOMPILED_TEMPLATE_NAME,
    SMS_TYPE,
)
from app.models import Notification, ServiceGuestList
from tests.app.db import (
    create_inbound_number,
    create_letter_contact,
    create_notification,
    create_reply_to_email,
    create_service,
    create_template,
    create_template_folder,
)

@pytest.mark.parametrize("mobile_number", ["07700 900678", "+44 7700 900678"])
def test_should_build_service_guest_list_from_mobile_number(mobile_number):
    service_guest_list = ServiceGuestList.from_string("service_id", MOBILE_TYPE, mobile_number)

    assert service_guest_list.recipient == mobile_number

@pytest.mark.parametrize("email_address", ["test@example.com"])
def test_should_build_service_guest_list_from_email_address(email_address):
    service_guest_list = ServiceGuestList.from_string("service_id", EMAIL_TYPE, email_address)

    assert service_guest_list.recipient == email_address

@pytest.mark.parametrize(
    "contact, recipient_type", [("", None), ("07700dsadsad", MOBILE_TYPE), ("gmail.com", EMAIL_TYPE)]
)
def test_should_not_build_service_guest_list_from_invalid_contact(recipient_type, contact):
    with pytest.raises(ValueError):
        ServiceGuestList.from_string("service_id", recipient_type, contact)

@pytest.mark.parametrize(
    "initial_statuses, expected_statuses",
    [
        # passing in single statuses as strings
        (NOTIFICATION_FAILED, NOTIFICATION_STATUS_TYPES_FAILED),
        (NOTIFICATION_STATUS_LETTER_ACCEPTED, [NOTIFICATION_SENDING, NOTIFICATION_CREATED]),
        (NOTIFICATION_CREATED, [NOTIFICATION_CREATED]),
        (NOTIFICATION_TECHNICAL_FAILURE, [NOTIFICATION_TECHNICAL_FAILURE]),
        # passing in lists containing single statuses
        ([NOTIFICATION_FAILED], NOTIFICATION_STATUS_TYPES_FAILED),
        ([NOTIFICATION_CREATED], [NOTIFICATION_CREATED]),
        ([NOTIFICATION_TECHNICAL_FAILURE], [NOTIFICATION_TECHNICAL_FAILURE]),
        (NOTIFICATION_STATUS_LETTER_RECEIVED, [NOTIFICATION_DELIVERED]),
        # passing in lists containing multiple statuses
        ([NOTIFICATION_FAILED, NOTIFICATION_CREATED], NOTIFICATION_STATUS_TYPES_FAILED + [NOTIFICATION_CREATED]),
        ([NOTIFICATION_CREATED, NOTIFICATION_PENDING], [NOTIFICATION_CREATED, NOTIFICATION_PENDING]),
        (
            [NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE],
            [NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE],
        ),
        (
            [NOTIFICATION_FAILED, NOTIFICATION_STATUS_LETTER_ACCEPTED],
            NOTIFICATION_STATUS_TYPES_FAILED + [NOTIFICATION_SENDING, NOTIFICATION_CREATED],
        ),
        # checking we don't end up with duplicates
        (
            [NOTIFICATION_FAILED, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE],
            NOTIFICATION_STATUS_TYPES_FAILED + [NOTIFICATION_CREATED],
        ),
    ],
)
def test_status_conversion(initial_statuses, expected_statuses):
    converted_statuses = Notification.substitute_status(initial_statuses)
    assert len(converted_statuses) == len(expected_statuses)
    assert set(converted_statuses) == set(expected_statuses)

@freeze_time("2016-01-01 11:09:00.000000")
@pytest.mark.parametrize(
    "template_type, recipient",
    [
        ("sms", "+447700900855"),
        ("email", "foo@bar.com"),
    ],
)
def test_notification_for_csv_returns_correct_type(sample_service, template_type, recipient):
    template = create_template(sample_service, template_type=template_type)
    notification = create_notification(template, to_field=recipient)

    serialized = notification.serialize_for_csv()
    assert serialized["template_type"] == template_type

@freeze_time("2016-01-01 11:09:00.000000")
def test_notification_for_csv_returns_correct_job_row_number(sample_job):
    notification = create_notification(sample_job.template, sample_job, job_row_number=0)

    serialized = notification.serialize_for_csv()
    assert serialized["row_number"] == 1

@freeze_time("2016-01-30 12:39:58.321312")
@pytest.mark.parametrize(
    "template_type, status, expected_status",
    [
        ("email", "failed", "Failed"),
        ("email", "technical-failure", "Technical failure"),
        ("email", "temporary-failure", "Inbox not accepting messages right now"),
        ("email", "permanent-failure", "Email address doesn’t exist"),
        ("sms", "temporary-failure", "Phone not accepting messages right now"),
        ("sms", "permanent-failure", "Phone number doesn’t exist"),
        ("sms", "sent", "Sent internationally"),
        ("letter", "created", "Accepted"),
        ("letter", "sending", "Accepted"),
        ("letter", "technical-failure", "Technical failure"),
        ("letter", "permanent-failure", "Permanent failure"),
        ("letter", "delivered", "Received"),
    ],
)
def test_notification_for_csv_returns_formatted_status(sample_service, template_type, status, expected_status):
    template = create_template(sample_service, template_type=template_type)
    notification = create_notification(template, status=status)

    serialized = notification.serialize_for_csv()
    assert serialized["status"] == expected_status

@freeze_time("2017-03-26 23:01:53.321312")
def test_notification_for_csv_returns_bst_correctly(sample_template):
    notification = create_notification(sample_template)

    serialized = notification.serialize_for_csv()
    assert serialized["created_at"] == "2017-03-27 00:01:53"

def test_notification_personalisation_getter_returns_empty_dict_from_None():
    noti = Notification()
    noti._personalisation = None
    assert noti.personalisation == {}

def test_notification_personalisation_getter_always_returns_empty_dict():
    noti = Notification()
    noti._personalisation = encryption.encrypt({})
    assert noti.personalisation == {}

@pytest.mark.parametrize("input_value", [None, {}])
def test_notification_personalisation_setter_always_sets_empty_dict(input_value):
    noti = Notification()
    noti.personalisation = input_value

    assert noti._personalisation == encryption.encrypt({})

def test_notification_subject_is_none_for_sms(sample_service):
    template = create_template(service=sample_service, template_type=SMS_TYPE)
    notification = create_notification(template=template)
    assert notification.subject is None

@pytest.mark.parametrize("template_type", ["email", "letter"])
def test_notification_subject_fills_in_placeholders(sample_service, template_type):
    template = create_template(service=sample_service, template_type=template_type, subject="((name))")
    notification = create_notification(template=template, personalisation={"name": "hello"})
    assert notification.subject == "hello"

def test_letter_notification_serializes_with_address(client, sample_letter_notification):
    sample_letter_notification.personalisation = {
        "address_line_1": "foo",
        "address_line_3": "bar",
        "address_line_5": None,
        "postcode": "SW1 1AA",
    }
    res = sample_letter_notification.serialize()
    assert res["line_1"] == "foo"
    assert res["line_2"] is None
    assert res["line_3"] == "bar"
    assert res["line_4"] is None
    assert res["line_5"] is None
    assert res["line_6"] is None
    assert res["postcode"] == "SW1 1AA"

def test_notification_serializes_created_by_name_with_no_created_by_id(client, sample_notification):
    res = sample_notification.serialize()
    assert res["created_by_name"] is None

def test_notification_serializes_created_by_name_with_created_by_id(client, sample_notification, sample_user):
    sample_notification.created_by_id = sample_user.id
    res = sample_notification.serialize()
    assert res["created_by_name"] == sample_user.name

def test_sms_notification_serializes_without_subject(client, sample_template):
    res = sample_template.serialize_for_v2()
    assert res["subject"] is None

def test_email_notification_serializes_with_subject(client, sample_email_template):
    res = sample_email_template.serialize_for_v2()
    assert res["subject"] == "Email Subject"

def test_letter_notification_serializes_with_subject(client, sample_letter_template):
    res = sample_letter_template.serialize_for_v2()
    assert res["subject"] == "Template subject"

def test_notification_references_template_history(client, sample_template):
    noti = create_notification(sample_template)
    sample_template.version = 3
    sample_template.content = "New template content"

    res = noti.serialize()
    assert res["template"]["version"] == 1

    assert res["body"] == noti.template.content
    assert noti.template.content != sample_template.content

def test_notification_requires_a_valid_template_version(client, sample_template):
    sample_template.version = 2
    with pytest.raises(IntegrityError):
        create_notification(sample_template)

def test_inbound_number_serializes_with_service(client, notify_db_session):
    service = create_service()
    inbound_number = create_inbound_number(number="1", service_id=service.id)
    serialized_inbound_number = inbound_number.serialize()
    assert serialized_inbound_number.get("id") == str(inbound_number.id)
    assert serialized_inbound_number.get("service").get("id") == str(inbound_number.service.id)
    assert serialized_inbound_number.get("service").get("name") == inbound_number.service.name

def test_inbound_number_returns_inbound_number(client, notify_db_session):
    service = create_service()
    inbound_number = create_inbound_number(number="1", service_id=service.id)

    assert service.get_inbound_number() == inbound_number.number

def test_inbound_number_returns_none_when_no_inbound_number(client, notify_db_session):
    service = create_service()

    assert not service.get_inbound_number()

def test_service_get_default_reply_to_email_address(sample_service):
    create_reply_to_email(service=sample_service, email_address="default@email.com")

    assert sample_service.get_default_reply_to_email_address() == "default@email.com"

def test_service_get_default_contact_letter(sample_service):
    create_letter_contact(service=sample_service, contact_block="London,\nNW1A 1AA")

    assert sample_service.get_default_letter_contact() == "London,\nNW1A 1AA"

def test_service_get_default_sms_sender(notify_db_session):
    service = create_service()
    assert service.get_default_sms_sender() == "testing"

def test_letter_notification_serializes_correctly(client, sample_letter_notification):
    sample_letter_notification.personalisation = {
        "addressline1": "test",
        "addressline2": "London",
        "postcode": "N1",
    }

    json = sample_letter_notification.serialize()
    assert json["line_1"] == "test"
    assert json["line_2"] == "London"
    assert json["postcode"] == "N1"

def test_letter_notification_postcode_can_be_null_for_precompiled_letters(client, sample_letter_notification):
    sample_letter_notification.personalisation = {
        "address_line_1": "test",
        "address_line_2": "London",
    }

    json = sample_letter_notification.serialize()
    assert json["line_1"] == "test"
    assert json["line_2"] == "London"
    assert json["postcode"] is None

def test_is_precompiled_letter_false(sample_letter_template):
    assert not sample_letter_template.is_precompiled_letter

def test_is_precompiled_letter_true(sample_letter_template):
    sample_letter_template.hidden = True
    sample_letter_template.name = PRECOMPILED_TEMPLATE_NAME
    assert sample_letter_template.is_precompiled_letter

def test_is_precompiled_letter_hidden_true_not_name(sample_letter_template):
    sample_letter_template.hidden = True
    assert not sample_letter_template.is_precompiled_letter

def test_is_precompiled_letter_name_correct_not_hidden(sample_letter_template):
    sample_letter_template.name = PRECOMPILED_TEMPLATE_NAME
    assert not sample_letter_template.is_precompiled_letter

def test_template_folder_is_parent(sample_service):
    x = None
    folders = []
    for i in range(5):
        x = create_template_folder(sample_service, name=str(i), parent=x)
        folders.append(x)

    assert folders[0].is_parent_of(folders[1])
    assert folders[0].is_parent_of(folders[2])
    assert folders[0].is_parent_of(folders[4])
    assert folders[1].is_parent_of(folders[2])
    assert not folders[1].is_parent_of(folders[0])

@pytest.mark.parametrize("is_platform_admin", (False, True))
def test_user_can_use_webauthn_if_platform_admin(sample_user, is_platform_admin):
    sample_user.platform_admin = is_platform_admin
    assert sample_user.can_use_webauthn == is_platform_admin

@pytest.mark.parametrize(
    ("auth_type", "can_use_webauthn"), [("email_auth", False), ("sms_auth", False), ("webauthn_auth", True)]
)
def test_user_can_use_webauthn_if_they_login_with_it(sample_user, auth_type, can_use_webauthn):
    sample_user.auth_type = auth_type
    assert sample_user.can_use_webauthn == can_use_webauthn

def test_user_can_use_webauthn_if_in_broadcast_org(sample_broadcast_service):
    assert sample_broadcast_service.users[0].can_use_webauthn

def test_user_can_use_webauthn_if_in_notify_team(notify_service):
    assert notify_service.users[0].can_use_webauthn


"""nathwani_erp URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.urls import path
from . import views
from django.contrib.auth.decorators import login_required

urlpatterns = [
    # path('admin/', admin.site.urls),
    path('module/add/', views.create_module),
    path('module/get/', login_required(views.ModuleListView.as_view())),
    path('admin/create_user/', views.create_user),
    path('admin/user_list/', views.user_list),
    path('admin/create_user_type/', views.create_user_type, name='create_user_type'),
    path('admin/get_users_type/', views.user_type_list),
    path('module/edit/', views.edit_module),
    path('module/edit_user/', views.edit_user),
    path('module/edit_user_type/', views.edit_user_type),
    path('module/delete_module', views.delete_module),
    path('module/delete_user', views.delete_user),
    path('module/delete_user_type', views.delete_user_type),
    path('profile/', views.profile),
    path('dashboard/', views.Dashboard.as_view(), name='dashboard'),
]


# -*- coding: utf-8 -*-

import os
import re
import time
import shelve
import thread
import base64
import bencode
import requests
import commands
import websocket

"""
On start following variables are set:
* 'masters' (type list) — contains Telegram user ids of users
who could control the bot.
* 'token' (type string) — Telegram bot API token.
* 'api_url' (type string) — Telegram bot API url + bot token.
* 'dl_dirs' (type dict) — bot sorts video files into two directories
"Series" and "Movies", all other fies goes to "General" downloads dir.
Paths could be changed according to your needs. Dict called from
on_ws_message() and dir_to_dl() in case you need to change 'keys'.
* 'pending_magnet' (type dict) — 'keys' are Telegram chat ids and
'values' are magnet links. Temporarely stores info to notify user
about download start. GID (download id) would be changet by aria2
after metadata downloaded and magnet link discarded.
* 'gid_chat' (type dict) — keeps aria2 download ids (GIDs) as 'keys'
and Telegram chat ids + download status as 'values'.
"""
masters, token = eval(open("token.cfg", "r").read())
api_url = "https://api.telegram.org/bot%s/" % token
dl_dirs = {"Series":"/var/plex/series",     # Directories to sort
           "Movies":"/var/plex/movies",     # and download files to.
           "General":"/home/nas/Downloads"} # Feel free to set yours.
pending_magnet = {}
gid_chat = {}
help_text = \
"""
This bot starts downloads by link or a .torrent file.
Sorts tv-shows, movies and general downloads apart automatically.
Accepts links and files from its master only.\n
*Commands:*
/special — specify download dir manually.
Usage: "/special download link, download dir"
/uptime — shows NAS uptime and CPU t°C
/status — shows current aria2 downloads status\n
More commands planned. ;)
"""

id_store = shelve.open("id_file", writeback=True)
'''
Opens shelve file. Saves 'gid_chat' and 'pending_magnet' to file
if file is empty. Or reads previous variables state from file if any.
'''
if not len(id_store) == 2:
  id_store['gid_chat'], id_store['pending_magnet'] = \
  gid_chat, pending_magnet
else:
  gid_chat, pending_magnet = \
  id_store['gid_chat'], id_store['pending_magnet']
id_store.close()

def conductor(chat_id, method, params):
  '''
  Sends all JSON requests over websocket messages
  to aria2 and logs them to file.
  '''
  request = ('{"jsonrpc":"2.0",'
           '"id":"%s",'
           '"method":"%s",'
           '"params":%s}'
           ) % (chat_id, method, params)
  wsocket.send(request)
  with open("nasbot_ws.log", "a") as logfile:
    if len(request) > 200:
      r = request[:200]
    else: r = request
    try:
      logfile.write("REQ: "
                    + time.strftime("%d/%m/%Y %H:%M:%S ")
                    + r.encode("utf-8"))
    except UnicodeDecodeError:
      logfile.write("REQ: "
                    + time.strftime("%d/%m/%Y %H:%M:%S ")
                    + r.encode('ascii', 'ignore'))
  os.system("echo \"$(tail -300 nasbot_ws.log)\"\
            > nasbot_ws.log")

def send_message(chat_id, text):
  '''
  Sends messages to Telegram user by chat id.
  Variable 'response' could be useful to log
  Telegram server responses, it could contain error messages if
  send fails.
  '''
  data = {"chat_id": chat_id, "text": text, "parse_mode": "Markdown"}
  response = requests.post(url=api_url + "sendMessage",
                           data=data).json()

def on_ws_message(wsocket, message):
  '''
  All websocket messages from aria2 handled here.
  Messages are JSONs, could contain 'id', 'gid' or both.
  Telegram chat ids + last known download status are used as 'id'.
  In case 'id' is not present Telegram chat id is retrevied by 'gid'
  from 'gid_chat'.
  '''
  response = eval(message.replace("\\" ,""))
  global gid_chat
  global pending_magnet
  global dl_dirs
  '''
  This spaghetti is pretty readable if you familiar with
  aria2 JSON responses. See the comments below.
  '''
  if 'id' in response and 'result' in response:
    '''
    This 'if' takes in aria2 JSONs like in the example below.
    Note that Telegram chat id is substituted with "1234567890"
    and aria2 gid in 'result' field is
    substituted with "SGVsbG8sIFdvcmxk":
    {"id":"1234567890-pending",
    "jsonrpc":"2.0",
    "result":"SGVsbG8sIFdvcmxk"}
    'id' utilised to store Telegram chat id and last known
    download state to keep track of every users downloads.
    '''
    if "-pending" in response['id'] or \
       "-sp_magnet" in response['id']:
      '''
      Saving gid and chat id ('id') as key: value pair to gid_chat.
      '''
      gid = response['result']
      gid_chat[gid] = response['id']
    elif "-done" in response['id']:
      '''
      After requesting aria2 with "method":"aria2.getFiles" and
      download gid ("params":["SGVsbG8sIFdvcmxk"] in our example)
      we'll get response with the list of files under "result" key
      even for finished downloads.
      '''
      for k, v in gid_chat.iteritems():
        if response['id'] == v:
          text = "*Downloaded files:*\n\n"
          for file in response['result']:
            text += (file['path'] + "\n\n")
          chat_id = v.replace("-done", "")
          for k in dl_dirs.keys():
            text = text.replace(dl_dirs[k],
                                "In *'"+ k + "'* directory: ")
          if len(text) >= 300:
            text = text[:text.find("\n\n", 300)] + "...\n\netc, etc"
          send_message(chat_id, text.replace("_", " "))
          del gid_chat[k]
    elif "-started" in response['id']:
      '''
      If download is started and "result" is not a list it is
      a string containing download's gid, so we have to request the
      files list from aria2.
      But if it is a list it contains file's URI for queued downloads
      and one-file downloads or the list of files and theirs os path
      for multi-file downloads (torrents).
      '''
      if not type(response['result']) is list:
        gid = response['result']
        gid_chat[gid] = response['id']
        conductor(gid_chat[gid], "aria2.getFiles", '["%s"]' % gid)
      else:
        try:
          if response['result'][0]['uris'][0]['status'] == "waiting":
            text = "*Download is queued.* Files waiting:\n\n%s" % \
                    response['result'][0]['uris']\
                    [0]['uri']
            send_message(response['id'].replace("-started", ""), text)
          elif response['result'][0]['uris'][0]['status'] == "used":
            text = "*File downloading:*\n\n%s" % \
                    response['result'][0]['uris']\
                    [0]['uri']
            send_message(response['id'].replace("-started", ""), text)
        except (IndexError, KeyError):
              text = "*Files downloading:*\n\n"
              for file in response['result']:
                text += (file['path'] + "\n\n")
              chat_id = response['id'].replace("-started", "")
              for k in dl_dirs.keys():
                text = text.replace(dl_dirs[k],
                                    "To *'"+ k + "'* directory: ")
              if len(text) >= 300:
                text = text[:text.find("\n\n", 300)]\
                       + "...\n\netc, etc"
              send_message(chat_id, text.replace("_", " "))
    elif "-fail" in response['id']:
      '''
      After requesting aria2 with "method":"aria2.getFiles" and
      download gid ("params":["SGVsbG8sIFdvcmxk"] in our example)
      we'll get response with the list of files under "result" key
      even for failed downloads.
      '''
      try:
        text = "*Download from link failed:*\n\n%s" % \
                  response['result'][0]['uris'][0]['uri']
        send_message(response['id'].replace("-fail", ""), text)
      except (IndexError, KeyError):
        text = "*Download of files failed:*\n\n"
        for file in response['result']:
          text += (file['path'] + "\n\n")
        chat_id = response['id'].replace("-fail", "")
        for k in dl_dirs.keys():
          text = text.replace(dl_dirs[k],
                              "To *'"+ k + "'* directory: ")
        if len(text) >= 300:
          text = text[:text.find("\n\n", 300)]\
                 + "...\n\netc, etc"
        send_message(chat_id, text.replace("_", " "))
    else:
      '''
      Just after collecting metadata for downloads
      started with magnet link aria2 changes download's gid
      and notifies us. We have to save new gid and chat id ('id')
      as key: value pair to gid_chat.
      '''
      gid_chat[response['result']] = response['id']
  elif 'id' in response and 'error' in response:
    '''
    Error is not always fatal. There is no "result" key in JSON,
    there is an "error" key instead.
    Download tracking stops for now and user is notified
    with error message.
    Telegram chat id + last known status saved in 'gid_chat' remains
    intact to notify the user about further status changes.
    Fatal errors are handled below in this code.
    '''
    position = response['id'].find("-sp_magnet")
    if position != -1:
      chat_id = response['id'][:position]
    else:
      chat_id = response['id'].replace("-started", "").replace(
                                       "-pending", "").replace(
                                       "-done", "")
    send_message(chat_id, response['error']['message'])
  elif response['method'] == "aria2.onDownloadComplete" or \
       response['method'] == "aria2.onBtDownloadComplete":
    '''
    Whenever download is started with magnet link aria2
    collects metadata first. It is saved as torrent file
    named with magnet link info hash. This name is saved to
    'pending_magnet'. On metadata download completion we've
    to start download with torrent file. So the filename from
    'pending_magnet' is passed to 'download_torrent()'
    in case we need to determine download folder automatically,
    or sent directly to aria2 via 'conductor()' in case it is
    a "/special" download and directory was predifined
    in the message.
    In case of torrent or file download completion
    we requesting aria2 with "method":"aria2.getFiles" and
    download gid ("params":["SGVsbG8sIFdvcmxk"] in our example).
    Note that there is no 'id' in this JSON, so
    the Telegram chat id is retrived by gid from 'gid_chat'.
    Here is JSON example:
    {"jsonrpc":"2.0",
    "method":"aria2.onDownloadComplete",
    "params":[{"gid":"SGVsbG8sIFdvcmxk"}]}
    '''
    gid = response['params'][0]['gid']
    if "-pending" in gid_chat[gid]:
      gid_chat[gid] = gid_chat[gid].replace("-pending", "")
      torrent_raw = open(pending_magnet[gid_chat[gid]]).read()
      download_torrent(gid_chat[gid] + "-started", torrent_raw)
      os.remove(pending_magnet[gid_chat[gid]])
      del pending_magnet[gid_chat[gid]]
      del gid_chat[gid]
    elif "-sp_magnet" in gid_chat[gid]:
      position = gid_chat[gid].find("-sp_magnet")
      download_dir = gid_chat[gid][position + 11:]
      gid_chat[gid] = gid_chat[gid][:position]
      try:
        torrent_b64 = base64.b64encode(open(pending_magnet\
                                    [gid_chat[gid]]).read())
      except (IndexError, AttributeError):
        send_message(gid_chat[gid],
                    "Torrent info is corrupted.")
      conductor(gid_chat[gid] + "-started", "aria2.addTorrent",
             '["%s", [], {"dir":"%s"}]' % (torrent_b64, download_dir))
      os.remove(pending_magnet[gid_chat[gid]])
      del pending_magnet[gid_chat[gid]]
      del gid_chat[gid]
    elif "-started" in gid_chat[gid]:
      gid_chat[gid] = gid_chat[gid].replace("-started", "-done")
      conductor(gid_chat[gid], "aria2.getFiles", '["%s"]' % gid)
  elif response['method'] == "aria2.onDownloadError":
    '''
    It is fatal error. Getting file info by gid and adding "-fail"
    status to the 'id' to notify the user on the next step.
    Note that there is no 'id' in this JSON, so
    the Telegram chat id is retrived by gid from 'gid_chat'.
    '''
    gid = response['params'][0]['gid']
    position = gid_chat[gid].find("-sp_magnet")
    if position != -1:
      gid_chat[gid] = gid_chat[gid][:position] + "-fail"
    else:
      gid_chat[gid] = gid_chat[gid].replace("-started", "-fail"
                                            ).replace(
                                            "-pending", "-fail"
                                            ).replace(
                                            "-done", "-fail")
    conductor(gid_chat[gid], "aria2.getFiles", '["%s"]' % gid)
    del gid_chat[gid]
  '''
  Here we saving current state to shelve file.
  '''
  id_store = shelve.open("id_file", writeback=True)
  id_store['gid_chat'], id_store['pending_magnet'] = \
  gid_chat, pending_magnet
  id_store.close()
  '''
  And writing the log.
  '''
  with open("nasbot_ws.log", "a") as logfile:
    try:
      logfile.write("MSG: "
                    + time.strftime("%d/%m/%Y %H:%M:%S ")
                    + message.encode("utf-8"))
    except UnicodeDecodeError:
      logfile.write("MSG: "
                    + time.strftime("%d/%m/%Y %H:%M:%S ")
                    + message.encode('ascii', 'ignore'))
  os.system("echo \"$(tail -300 nasbot_ws.log)\"\
            > nasbot_ws.log")

def on_ws_error(wsocket, error):
  """
  TODO: depending on flavour of error have to
  send error msg to master or log it to file.
  For now just log everything.
  """
  with open("nasbot_ws.log", "a") as logfile:
    try:
      logfile.write("ERR: "
                    + time.strftime("%d/%m/%Y %H:%M:%S")
                    + error.encode("utf-8"))
    except UnicodeDecodeError:
      logfile.write("ERR: "
                    + time.strftime("%d/%m/%Y %H:%M:%S ")
                    + error.encode('ascii', 'ignore'))
  os.system("echo \"$(tail -300 nasbot_ws.log)\"\
            > nasbot_ws.log")

def on_ws_close(wsocket):
  '''
  Restart websocket connection if it is closed.
  No error handling, just keep trying.
  '''
  thread.start_new_thread(wsocket.run_forever, ())

'''
Starting websocket connection in the new thread.
'''
wsocket = websocket.WebSocketApp("ws://localhost:6800/jsonrpc",
                                  on_message = on_ws_message,
                                  on_error = on_ws_error,
                                  on_close = on_ws_close)

thread.start_new_thread(wsocket.run_forever, ())

def get_updates(offset):
  '''
  Another plate of spaghetti.
  '''
  updates = []
  try:
    '''
    Telegram bot messages are basically JSONs.
    Parsing chat id ['message']['chat']['id'],
    user id ['message']['from'] and text ['message']['text']
    or torrent file id ['message']['document']['file_id']
    out of 'result'.
    '''
    for msg in requests.get(api_url
                            + "getUpdates?timeout=100"
                            + offset).json()['result']:
      if 'message' in msg and \
         'text' in msg['message']:
        msg_text = msg['message']['text']
        user_id = msg['message']['from']
        chat_id = str(msg['message']['chat']['id'])
        '''
        Checking Telegram bot "commands" in texts. If no commands
        found appending 'updates' type list with another list
        containing chat id, user id, and message text.
        Example result:
        [["1234567890", "1234567890", "Hello Robot!"]].
        '''
        if msg_text == "/help":
          global help_text
          send_message(chat_id, help_text)
        elif msg_text == "/status" and \
             str(user_id['id']) in masters:
          try:
            stat_command = commands.getoutput(
                    "tail -100 aria2supervised.out.log").rsplit(
                    "*** Download Progress Summary ", 1)[1].rsplit(
                    "-"*79, 1)[0]
            stat = "*Download Progress Summary*\n"\
                   + stat_command.replace("***", "").replace(
                                          "#", "").replace(
                                          "_", "").replace(
                                          "="*79, "").replace(
                                          "-"*79, "")
          except IndexError:
            '''
            We'll get IndexError if aria2 was shutdown and lost all
            downloads/uploads data or all downloads are finished and
            removed from active. Just can't split the string because
            there are no such substrings in recent log.
            '''
            stat = "There are no active downloads/uploads."
          send_message(chat_id, stat)
        elif msg_text == "/uptime" and \
             str(user_id['id']) in masters:
          upt_command = "uptime | grep -o 'up.*'\
                        | sed 's/up/Uptime:/g'\
                        ; sensors | grep -o 'Core.\{0,19\}'\
                        | sed 's/  //g'"
          send_message(chat_id, commands.getoutput(upt_command))
        elif msg_text[:8] == "/special" and \
             str(user_id['id']) in masters:
          try:
            attr_list = msg_text[8:].replace(" ","").split(",")
            attr_list.insert(1, dl_dirs['General'])
            if attr_list[0][-8:] == ".torrent":
              try:
                attr_list[0] = \
                base64.b64encode(requests.get(attr_list[0],
                                              stream=True).raw.read())
                conductor(chat_id + "-started", "aria2.addTorrent",
                 '["%s", [], {"dir":"%s/%s"}]' % tuple(attr_list))
                send_message(chat_id, "Collecting metadata.")
              except (IndexError, AttributeError):
                send_message(chat_id,
                            "Torrent info is corrupted.")
            elif attr_list[0][:8] == "magnet:?":
              id_store = shelve.open("id_file", writeback=True)
              try:
                pending_magnet[chat_id] = re.search(
                                          r"btih:(\w+)&?",
                                          attr_list[0]
                                          ).group(1).lower() \
                                          + ".torrent"
                conductor(chat_id + "-sp_magnet:" + attr_list[1] + "/"
                          + attr_list[2], "aria2.addUri",
                          '[["%s"], {"bt-metadata-only":"true",'\
                          '"bt-save-metadata":"true"}]'
                          % attr_list[0])
                send_message(chat_id, "Collecting metadata.")
              except AttributeError:
                send_message(chat_id,
                             "*It's not a magnet link:*\n\n%s"
                             % attr_list[0])
              id_store['pending_magnet'] = pending_magnet
              id_store.close()
            else:
              conductor(chat_id + "-started", "aria2.addUri",
               '[["%s"], {"dir":"%s/%s"}]' % tuple(attr_list))
          except TypeError:
            send_message(chat_id,
                        "Something wrong, maybe /help is needed?")
        else:
          updates.append([ chat_id, user_id, msg_text ])
      elif 'message' in msg and \
           'document' in msg['message'] and \
            str(msg['message']['from']['id']) in masters:
        if msg['message']['document']['mime_type'] == \
           "application/x-bittorrent":
          uri = "https://api.telegram.org/file/bot%s/"\
                % token \
                + requests.get(
                  api_url
                  + "getFile?file_id="
                  + msg['message']['document']['file_id']
                  ).json()['result']['file_path']
          download_torrent(str(msg['message']['chat']['id'])
                           + "-started",
                           uri, uri=True)
      offset = "&offset=%d" % (msg['update_id'] + 1)
  except (ValueError, requests.exceptions.ConnectionError):
    return [], offset
  return updates, offset

def dir_to_dl(link):
  global dl_dirs
  if re.findall(r"\.(avi|wmv|mkv|mov|m4v|mp4|mpeg|mpg)$",
                link, re.IGNORECASE|re.MULTILINE):
    if re.search(r"season|сезон|s\d\d?e\d\d?",
                 link, re.IGNORECASE|re.MULTILINE):
      download_dir = dl_dirs['Series']
    else:
      download_dir = dl_dirs['Movies']
  else:
    download_dir = dl_dirs['General']
  return download_dir

def download_torrent(chat_id, data, uri=False):
  if uri:
    torrent_raw = requests.get(data,
                  stream=True).raw.read()
  else: torrent_raw = data
  """
  gave up to parse files from bencoded string by myself
  so imported bencode module
  """
  try:
    torrent_b64 = base64.b64encode(torrent_raw)
    torrent_info = bencode.bdecode(torrent_raw)['info']
    files_list = ""
    if 'files' in torrent_info:
      for file in torrent_info['files']:
        files_list += str("\\".join(file['path'])+"\n")
    else: files_list = str(torrent_info['name'])
    download_dir = dir_to_dl(files_list)
    conductor(chat_id, "aria2.addTorrent",
             '["%s", [], {"dir":"%s"}]' % (torrent_b64, download_dir))
  except (IndexError, AttributeError):
    send_message(chat_id.replace("-started", ""),
                "Torrent info is corrupted.")

def parse_uri(updates):
  for msg in updates:
    if str(msg[1]['id']) in masters:
      uri = re.search(
            r"^(magnet\S+)|(\S\.torrent)$|^(http\S+|ftp\S+|sftp\S+)",
            msg[2])
      if uri:
        if uri.group(1):
          global pending_magnet
          id_store = shelve.open("id_file", writeback=True)
          try:
            pending_magnet[msg[0]] = re.search(r"btih:(\w+)&?",
                                     uri.group(1)).group(1).lower() \
                                     + ".torrent"
            conductor(msg[0] + "-pending", "aria2.addUri",
                      '[["%s"], {"bt-metadata-only":"true",'\
                      '"bt-save-metadata":"true"}]' % uri.group(1))
            send_message(msg[0], "Collecting metadata.")
          except AttributeError:
            send_message(msg[0], "*It's not a magnet link:*\n\n%s."
                        % uri.group(1))
          id_store['pending_magnet'] = pending_magnet
          id_store.close()

        elif uri.group(2):
          download_torrent(msg[0] + "-started",
                           uri.group(2), uri=True)
        elif uri.group(3) and not uri.group(2):
          download_dir = dir_to_dl(uri.group(3))
          conductor(msg[0] + "-started", "aria2.addUri",
                    '[["%s"], {"dir":"%s"}]' % (msg[2], download_dir))
      else:
        send_message(msg[0], "I can't do that.")
    else:
      send_message(msg[0], "Human, you'r not my master.")
      try:
        with open("nasbot_nonusers.log", "a") as logfile:
          logfile.write(msg.encode("utf-8"))
        os.system("echo \"$(tail -300 nasbot_nonusers.log)\"\
                  > nasbot_nonusers.log")
      except UnicodeDecodeError:
        pass

def main():
  offset = ""
  while True:
    updates, offset = get_updates(offset)
    if len(updates) > 0:
      parse_uri(updates)
    time.sleep(5)

if __name__ == '__main__':
    main()


from BiSheServer.settings import CONFIG
from api import user_api, api
from api.redis_pool import cache
from rest_framework.views import APIView
from api.response import JsonError, JsonResponse
from user.models import UsersBase
from .email import send_reg_email as send_email
from django.db.models import Q
import string, random, time, re

class send_reg_email(APIView):
    def get(self, request, *args, **kwargs):
        return JsonError("不支持Get请求！")

    def post(self, request, *args, **kwargs):
        user_email = request.POST.get("email")

        match_mail = re.compile(r'^[A-Za-z\d]+([-_.][A-Za-z\d]+)*@([A-Za-z\d]+[-.])+[A-Za-z\d]{2,4}$')
        if not match_mail.match(user_email) or not user_email:
            return JsonError("邮箱格式不正确，请检查！")

        try:
            send_type = int(request.GET.get("type"))
        except:
            return JsonError("请求参数不正确！")

        # cache.set('email_verify_code',email_verify_code)
        # cache.expire('email_verify_code', 60 * 3)
        # request.session['email_verify_code'].set_expiry(60*3)
        email_verify_code = "".join(random.sample([x for x in string.digits], 6))

        if send_type == 1:  # 注册验证
            try:
                if request.session['user_reg_step'] != 1:
                    raise Exception
            except:
                return JsonError("请先完成第一步注册！")
            if UsersBase.objects.filter(Q(user_mail=user_email)).exclude(user_reg_step=1).exists():
                return JsonError("邮箱已存在，请更换！")
            title = "注册验证邮件"
            request.session['email_verify_code'] = [email_verify_code, time.time()]  # code加入到session中
            contents = '

您的验证码为：

 \
            '+email_verify_code+'

有效期3分钟，感谢您的注册加入。 \

'
            request.session['user_email'] = user_email
        elif send_type == 2:  # 忘记密码验证
            try:
                if request.session['fget_step_code'] != 1:
                    raise Exception
            except:
                return JsonError("请先完成第一步操作！")
            if request.session['user_fget_email'] != user_email:
                return JsonError("邮箱不正确！")
            title = "忘记密码验证邮件"
            request.session['email_fget_verify_code'] = [email_verify_code, time.time()]  # code加入到session中
            contents = '

您的验证码为：

 \
            '+email_verify_code+'

有效期3分钟，感谢您的使用。 \

'
        elif send_type == 3:  # 更改邮箱验证
            if user_api.User().isNotLogin(request):
                return JsonError("请先登录！")

            randCodeVailRs = api.Api().randCodeVail(request, request.POST.get("randCode"))
            if not randCodeVailRs[0]:
                return JsonError(randCodeVailRs[1])

            if UsersBase.objects.filter(Q(user_mail=user_email)).exists():
                return JsonError("邮箱已存在，请更换！")
            title = "更改邮箱信息验证邮件"
            request.session['email_modify_verify_code'] = [email_verify_code, time.time()]  # code加入到session中
            contents = '

您的验证码为：

 \
            ' + email_verify_code + '

有效期3分钟，感谢您的使用。 \

'
        else:
            return JsonError("请求参数不正确！")

        if not cache.get(user_email):
            cache.set(user_email, 1)
            cache.expire(user_email, 60 * 60 * 24)
        elif int(cache.get(user_email).decode('utf-8')) <= 2:
            cache.set(user_email, int(cache.get(user_email).decode('utf-8')) + 1, cache.ttl(user_email))
        else:
            return JsonError("该邮箱发送次数已超限制，请改日再试！")

        if CONFIG.get('EMAIL', 'EMAIL_USE') == 'False':
            return JsonResponse({"msg": "验证码发送成功！邮箱验证码为：" + str(email_verify_code)})

        try:
            send_email(user_email, title, contents)
        except Exception as e:
            # print(e)
            return JsonError("发送失败，请确认邮箱地址正确后重试！")
        return JsonResponse({"msg": "验证码发送成功！"})