import os
import json
import sqlite3
import argparse
import logging
import pickle
from copy import deepcopy
import difflib
import traceback

from semparser.common import registry
from semparser.common.utils import print_dict

from semparser.modules.semantic_parser.preprocessor.process_spider_sql import get_sql, get_schema
from semparser.modules.semantic_parser.asdl.spider.spider_hypothesis import SpiderDecodeHypothesis
from semparser.modules.semantic_parser.inference.spider_ast import SpiderAST

# Flag to disable value evaluation
DISABLE_VALUE = True
# Flag to disable distinct in select evaluation
DISABLE_DISTINCT = True

CLAUSE_KEYWORDS = ('select', 'from', 'where', 'group', 'order',
                   'limit', 'intersect', 'union', 'except')
JOIN_KEYWORDS = ('join', 'on', 'as')

WHERE_OPS = ('not', 'between', '=', '>', '<', '>=', '<=', '!=', 'in', 'like', 'is', 'exists')
UNIT_OPS = ('none', '-', '+', "*", '/')
AGG_OPS = ('none', 'max', 'min', 'count', 'sum', 'avg')
TABLE_TYPE = {
    'sql': "sql",
    'table_unit': "table_unit",
}

COND_OPS = ('and', 'or')
SQL_OPS = ('intersect', 'union', 'except')
ORDER_OPS = ('desc', 'asc')

HARDNESS = {
    "component1": ('where', 'group', 'order', 'limit', 'join', 'or', 'like'),
    "component2": ('except', 'union', 'intersect')
}

class Schema:
    def __init__(self, schema):
        self._schema = schema
        self._idMap = self._map(self._schema)

    @property
    def schema(self):
        return self._schema

    @property
    def idMap(self):
        return self._idMap

    def _map(self, schema):
        idMap = {'*': "__all__"}
        id = 1
        for key, vals in schema.iteritems():
            for val in vals:
                idMap[key.lower() + "." + val.lower()] = "__" + \
                    key.lower() + "." + val.lower() + "__"
                id += 1

        for key in schema:
            idMap[key.lower()] = "__" + key.lower() + "__"

        return idMap

def condition_has_or(conds):
    return 'or' in conds[1::2]

def condition_has_like(conds):
    return WHERE_OPS.index('like') in [cond_unit[1] for cond_unit in conds[::2]]

def condition_has_sql(conds):
    for cond_unit in conds[::2]:
        val1, val2 = cond_unit[3], cond_unit[4]
        if val1 is not None and type(val1) is dict:
            return True
        if val2 is not None and type(val2) is dict:
            return True
    return False

def val_has_op(val_unit):
    return val_unit[0] != UNIT_OPS.index('none')

def has_agg(unit):
    return unit[0] != AGG_OPS.index('none')

def accuracy(count, total):
    if count == total:
        return 1
    return 0

def recall(count, total):
    if count == total:
        return 1
    return 0

def F1(acc, rec):
    if (acc + rec) == 0:
        return 0
    return (2. * acc * rec) / (acc + rec)

def get_scores(count, pred_total, label_total):
    if pred_total != label_total:
        return 0, 0, 0
    elif count == pred_total:
        return 1, 1, 1
    return 0, 0, 0

def eval_sel(pred, label):
    pred_sel = pred['select'][1]
    label_sel = [x for x in label['select'][1]]
    label_wo_agg = [unit[1] for unit in label_sel]
    pred_total = len(pred_sel)
    label_total = len(label_sel)
    cnt = 0
    cnt_wo_agg = 0

    for unit in pred_sel:
        if unit in label_sel:
            cnt += 1
            label_sel.remove(unit)
        if unit[1] in label_wo_agg:
            cnt_wo_agg += 1
            label_wo_agg.remove(unit[1])

    return label_total, pred_total, cnt, cnt_wo_agg

def eval_where(pred, label):
    pred_conds = [unit for unit in pred['where'][::2]]
    label_conds = [unit for unit in label['where'][::2]]
    label_wo_agg = [unit[2] for unit in label_conds]
    pred_total = len(pred_conds)
    label_total = len(label_conds)
    cnt = 0
    cnt_wo_agg = 0

    for unit in pred_conds:
        if unit in label_conds:
            cnt += 1
            label_conds.remove(unit)
        if unit[2] in label_wo_agg:
            cnt_wo_agg += 1
            label_wo_agg.remove(unit[2])

    return label_total, pred_total, cnt, cnt_wo_agg

def eval_group(pred, label):
    pred_cols = [unit[1] for unit in pred['groupBy']]
    label_cols = [unit[1] for unit in label['groupBy']]
    pred_total = len(pred_cols)
    label_total = len(label_cols)
    cnt = 0
    pred_cols = [pred for pred in pred_cols]
    label_cols = [label for label in label_cols]
    for col in pred_cols:
        if col in label_cols:
            cnt += 1
            label_cols.remove(col)
    return label_total, pred_total, cnt

def eval_having(pred, label):
    pred_total = label_total = cnt = 0
    if len(pred['groupBy']) > 0:
        pred_total = 1
    if len(label['groupBy']) > 0:
        label_total = 1

    pred_cols = [unit[1] for unit in pred['groupBy']]
    label_cols = [unit[1] for unit in label['groupBy']]
    if pred_total == label_total == 1 \
            and pred_cols == label_cols \
            and pred['having'] == label['having']:
        cnt = 1

    return label_total, pred_total, cnt

def eval_order(pred, label):
    pred_total = label_total = cnt = 0
    if len(pred['orderBy']) > 0:
        pred_total = 1
    if len(label['orderBy']) > 0:
        label_total = 1
    if len(label['orderBy']) > 0 and pred['orderBy'] == label['orderBy'] and (
            (pred['limit'] is None and label['limit'] is None) or (
                pred['limit'] is not None and label['limit'] is not None)):
        cnt = 1
    return label_total, pred_total, cnt

def eval_and_or(pred, label):
    pred_ao = pred['where'][1::2]
    label_ao = label['where'][1::2]
    pred_ao = set(pred_ao)
    label_ao = set(label_ao)

    if pred_ao == label_ao:
        return 1, 1, 1
    return len(pred_ao), len(label_ao), 0

def get_nestedSQL(sql):
    nested = []
    for cond_unit in sql['from']['conds'][::2] + sql['where'][::2] + sql['having'][::2]:
        if type(cond_unit[3]) is dict:
            nested.append(cond_unit[3])
        if type(cond_unit[4]) is dict:
            nested.append(cond_unit[4])
    if sql['intersect'] is not None:
        nested.append(sql['intersect'])
    if sql['except'] is not None:
        nested.append(sql['except'])
    if sql['union'] is not None:
        nested.append(sql['union'])
    return nested

def eval_nested(pred, label):
    label_total = 0
    pred_total = 0
    cnt = 0
    if pred is not None:
        pred_total += 1
    if label is not None:
        label_total += 1
    if pred is not None and label is not None:
        cnt += SpiderEvaluator().eval_exact_match(pred, label)
    return label_total, pred_total, cnt

def eval_IUEN(pred, label):
    lt1, pt1, cnt1 = eval_nested(pred['intersect'], label['intersect'])
    lt2, pt2, cnt2 = eval_nested(pred['except'], label['except'])
    lt3, pt3, cnt3 = eval_nested(pred['union'], label['union'])
    label_total = lt1 + lt2 + lt3
    pred_total = pt1 + pt2 + pt3
    cnt = cnt1 + cnt2 + cnt3
    return label_total, pred_total, cnt

def get_keywords(sql):
    res = set()
    if len(sql['where']) > 0:
        res.add('where')
    if len(sql['groupBy']) > 0:
        res.add('group')
    if len(sql['having']) > 0:
        res.add('having')
    if len(sql['orderBy']) > 0:
        res.add(sql['orderBy'][0])
        res.add('order')
    if sql['limit'] is not None:
        res.add('limit')
    if sql['except'] is not None:
        res.add('except')
    if sql['union'] is not None:
        res.add('union')
    if sql['intersect'] is not None:
        res.add('intersect')

    # or keyword
    ao = sql['from']['conds'][1::2] + sql['where'][1::2] + sql['having'][1::2]
    if len([token for token in ao if token == 'or']) > 0:
        res.add('or')

    cond_units = sql['from']['conds'][::2] + sql['where'][::2] + sql['having'][::2]
    # not keyword
    if len([cond_unit for cond_unit in cond_units if cond_unit[0]]) > 0:
        res.add('not')

    # in keyword
    if len([cond_unit for cond_unit in cond_units if cond_unit[1] == WHERE_OPS.index('in')]) > 0:
        res.add('in')

    # like keyword
    if len([cond_unit for cond_unit in cond_units if cond_unit[1] == WHERE_OPS.index('like')]) > 0:
        res.add('like')

    return res

def eval_keywords(pred, label):
    pred_keywords = get_keywords(pred)
    label_keywords = get_keywords(label)
    pred_total = len(pred_keywords)
    label_total = len(label_keywords)
    cnt = 0

    for k in pred_keywords:
        if k in label_keywords:
            cnt += 1
    return label_total, pred_total, cnt

def count_agg(units):
    return len([unit for unit in units if has_agg(unit)])

def count_component1(sql):
    count = 0
    if len(sql['where']) > 0:
        count += 1
    if len(sql['groupBy']) > 0:
        count += 1
    if len(sql['orderBy']) > 0:
        count += 1
    if sql['limit'] is not None:
        count += 1
    if len(sql['from']['table_units']) > 0:  # JOIN
        count += len(sql['from']['table_units']) - 1

    ao = sql['from']['conds'][1::2] + sql['where'][1::2] + sql['having'][1::2]
    count += len([token for token in ao if token == 'or'])
    cond_units = sql['from']['conds'][::2] + sql['where'][::2] + sql['having'][::2]
    count += len([cond_unit for cond_unit in cond_units if cond_unit[1] == WHERE_OPS.index('like')])

    return count

def count_component2(sql):
    nested = get_nestedSQL(sql)
    return len(nested)

def count_others(sql):
    count = 0
    # number of aggregation
    agg_count = count_agg(sql['select'][1])
    agg_count += count_agg(sql['where'][::2])
    agg_count += count_agg(sql['groupBy'])
    if len(sql['orderBy']) > 0:
        agg_count += count_agg([unit[1] for unit in sql['orderBy'][1] if unit[1]] + [
            unit[2] for unit in sql['orderBy'][1] if unit[2]])
    agg_count += count_agg(sql['having'])
    if agg_count > 1:
        count += 1

    # number of select columns
    if len(sql['select'][1]) > 1:
        count += 1

    # number of where conditions
    if len(sql['where']) > 1:
        count += 1

    # number of group by clauses
    if len(sql['groupBy']) > 1:
        count += 1

    return count

class SpiderEvaluator:
    """A simple evaluator"""

    def __init__(self):
        self.partial_scores = None

    def eval_hardness(self, sql):
        count_comp1_ = count_component1(sql)
        count_comp2_ = count_component2(sql)
        count_others_ = count_others(sql)

        if count_comp1_ <= 1 and count_others_ == 0 and count_comp2_ == 0:
            return "easy"
        elif (count_others_ <= 2 and count_comp1_ <= 1 and count_comp2_ == 0) or \
                (count_comp1_ <= 2 and count_others_ < 2 and count_comp2_ == 0):
            return "medium"
        elif (count_others_ > 2 and count_comp1_ <= 2 and count_comp2_ == 0) or \
                (2 < count_comp1_ <= 3 and count_others_ <= 2 and count_comp2_ == 0) or \
                (count_comp1_ <= 1 and count_others_ == 0 and count_comp2_ <= 1):
            return "hard"
        else:
            return "extra"

    def eval_exact_match(self, pred, label):
        partial_scores = self.eval_partial_match(pred, label)
        self.partial_scores = partial_scores

        for x, score in partial_scores.items():
            if score['f1'] != 1:
                return 0
        if len(label['from']['table_units']) > 0:
            label_tables = sorted(label['from']['table_units'])
            pred_tables = sorted(pred['from']['table_units'])
            return label_tables == pred_tables
        return 1

    def eval_partial_match(self, pred, label):
        res = {}

        label_total, pred_total, cnt, cnt_wo_agg = eval_sel(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['select'] = {'acc': acc, 'rec': rec, 'f1': f1,
                         'label_total': label_total, 'pred_total': pred_total}
        acc, rec, f1 = get_scores(cnt_wo_agg, pred_total, label_total)
        res['select(no AGG)'] = {'acc': acc, 'rec': rec, 'f1': f1,
                                 'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt, cnt_wo_agg = eval_where(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['where'] = {'acc': acc, 'rec': rec, 'f1': f1,
                        'label_total': label_total, 'pred_total': pred_total}
        acc, rec, f1 = get_scores(cnt_wo_agg, pred_total, label_total)
        res['where(no OP)'] = {'acc': acc, 'rec': rec, 'f1': f1,
                               'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt = eval_group(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['group(no Having)'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total,
                                   'pred_total': pred_total}

        label_total, pred_total, cnt = eval_having(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['group'] = {'acc': acc, 'rec': rec, 'f1': f1,
                        'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt = eval_order(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['order'] = {'acc': acc, 'rec': rec, 'f1': f1,
                        'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt = eval_and_or(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['and/or'] = {'acc': acc, 'rec': rec, 'f1': f1,
                         'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt = eval_IUEN(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['IUEN'] = {'acc': acc, 'rec': rec, 'f1': f1,
                       'label_total': label_total, 'pred_total': pred_total}

        label_total, pred_total, cnt = eval_keywords(pred, label)
        acc, rec, f1 = get_scores(cnt, pred_total, label_total)
        res['keywords'] = {'acc': acc, 'rec': rec, 'f1': f1,
                           'label_total': label_total, 'pred_total': pred_total}

        return res

def isValidSQL(sql, db):
    conn = sqlite3.connect(db)
    cursor = conn.cursor()
    try:
        cursor.execute(sql)
    except Exception:
        return False
    return True

def print_scores(scores, etype, p_func=print):
    levels = ['easy', 'medium', 'hard', 'extra', 'all']
    partial_types = ['select', 'select(no AGG)', 'where', 'where(no OP)', 'group(no Having)',
                     'group', 'order', 'and/or', 'IUEN', 'keywords']

    p_func("{:20} {:20} {:20} {:20} {:20} {:20}".format("", *levels))
    counts = [scores[level]['count'] for level in levels]
    p_func("{:20} {:<20d} {:<20d} {:<20d} {:<20d} {:<20d}".format("count", *counts))

    if etype in ["all", "exec"]:
        p_func('=====================   EXECUTION ACCURACY     =====================')
        this_scores = [scores[level]['exec'] for level in levels]
        p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(
            "execution", *this_scores))

    if etype in ["all", "match"]:
        p_func('\n====================== EXACT MATCHING ACCURACY =====================')
        exact_scores = [scores[level]['exact'] for level in levels]
        p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(
            "exact match", *exact_scores))

        if 'exact_alan' in scores['all']:
            exact_scores_alan = [scores[level]['exact_alan'] for level in levels]
            p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(
                "Alan exact match", *exact_scores_alan))

        p_func('\n---------------------PARTIAL MATCHING ACCURACY----------------------')
        for type_ in partial_types:
            this_scores = [scores[level]['partial'][type_]['acc'] for level in levels]
            p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(type_, *this_scores))

        p_func('---------------------- PARTIAL MATCHING RECALL ----------------------')
        for type_ in partial_types:
            this_scores = [scores[level]['partial'][type_]['rec'] for level in levels]
            p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(type_, *this_scores))

        p_func('---------------------- PARTIAL MATCHING F1 --------------------------')
        for type_ in partial_types:
            this_scores = [scores[level]['partial'][type_]['f1'] for level in levels]
            p_func("{:20} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f} {:<20.3f}".format(type_, *this_scores))

def evaluate(gold, predict, db_dir, etype, kmaps):
    with open(gold) as f:
        glist = [l.strip().split('\t') for l in f.readlines() if len(l.strip()) > 0]

    with open(predict) as f:
        plist = [l.strip().split('\t') for l in f.readlines() if len(l.strip()) > 0]
    evaluator = SpiderEvaluator()

    levels = ['easy', 'medium', 'hard', 'extra', 'all']
    partial_types = ['select', 'select(no AGG)', 'where', 'where(no OP)', 'group(no Having)',
                     'group', 'order', 'and/or', 'IUEN', 'keywords']
    entries = []
    scores = {}

    for level in levels:
        scores[level] = {'count': 0, 'partial': {}, 'exact': 0.}
        scores[level]['exec'] = 0
        for type_ in partial_types:
            scores[level]['partial'][type_] = {'acc': 0.,
                                               'rec': 0., 'f1': 0., 'acc_count': 0, 'rec_count': 0}

    eval_err_num = 0
    for p, g in zip(plist, glist):
        p_str = p[0]
        g_str, db = g
        db_name = db
        db = os.path.join(db_dir, db, db + ".sqlite")
        schema = Schema(get_schema(db))
        g_sql = get_sql(schema, g_str)
        hardness = evaluator.eval_hardness(g_sql)
        scores[hardness]['count'] += 1
        scores['all']['count'] += 1

        try:
            p_sql = get_sql(schema, p_str)
        except Exception:
            # If p_sql is not valid, then we will use an empty sql to evaluate with the correct sql
            p_sql = {
                "except": None,
                "from": {
                    "conds": [],
                    "table_units": []
                },
                "groupBy": [],
                "having": [],
                "intersect": None,
                "limit": None,
                "orderBy": [],
                "select": [
                    False,
                    []
                ],
                "union": None,
                "where": []
            }
            eval_err_num += 1
            print("eval_err_num:{}".format(eval_err_num))

        # rebuild sql for value evaluation
        kmap = kmaps[db_name]
        g_valid_col_units = build_valid_col_units(g_sql['from']['table_units'], schema)
        g_sql = rebuild_sql_val(g_sql)
        g_sql = rebuild_sql_col(g_valid_col_units, g_sql, kmap)
        p_valid_col_units = build_valid_col_units(p_sql['from']['table_units'], schema)
        p_sql = rebuild_sql_val(p_sql)
        p_sql = rebuild_sql_col(p_valid_col_units, p_sql, kmap)

        if etype in ["all", "exec"]:
            exec_score = eval_exec_match(db, p_str, g_str, p_sql, g_sql)
            if exec_score:
                scores[hardness]['exec'] += 1

        if etype in ["all", "match"]:
            exact_score = evaluator.eval_exact_match(p_sql, g_sql)
            partial_scores = evaluator.partial_scores
            if exact_score == 0:
                print("{} pred: {}".format(hardness, p_str))
                print("{} gold: {}".format(hardness, g_str))
                print("")
            scores[hardness]['exact'] += exact_score
            scores['all']['exact'] += exact_score
            for type_ in partial_types:
                if partial_scores[type_]['pred_total'] > 0:
                    scores[hardness]['partial'][type_]['acc'] += partial_scores[type_]['acc']
                    scores[hardness]['partial'][type_]['acc_count'] += 1
                if partial_scores[type_]['label_total'] > 0:
                    scores[hardness]['partial'][type_]['rec'] += partial_scores[type_]['rec']
                    scores[hardness]['partial'][type_]['rec_count'] += 1
                scores[hardness]['partial'][type_]['f1'] += partial_scores[type_]['f1']
                if partial_scores[type_]['pred_total'] > 0:
                    scores['all']['partial'][type_]['acc'] += partial_scores[type_]['acc']
                    scores['all']['partial'][type_]['acc_count'] += 1
                if partial_scores[type_]['label_total'] > 0:
                    scores['all']['partial'][type_]['rec'] += partial_scores[type_]['rec']
                    scores['all']['partial'][type_]['rec_count'] += 1
                scores['all']['partial'][type_]['f1'] += partial_scores[type_]['f1']

            entries.append({
                'predictSQL': p_str,
                'goldSQL': g_str,
                'hardness': hardness,
                'exact': exact_score,
                'partial': partial_scores
            })

    for level in levels:
        if scores[level]['count'] == 0:
            continue
        if etype in ["all", "exec"]:
            scores[level]['exec'] /= scores[level]['count']

        if etype in ["all", "match"]:
            scores[level]['exact'] /= scores[level]['count']
            for type_ in partial_types:
                if scores[level]['partial'][type_]['acc_count'] == 0:
                    scores[level]['partial'][type_]['acc'] = 0
                else:
                    scores[level]['partial'][type_]['acc'] = scores[level]['partial'][type_]['acc'] / \
                        scores[level]['partial'][type_]['acc_count'] * 1.0
                if scores[level]['partial'][type_]['rec_count'] == 0:
                    scores[level]['partial'][type_]['rec'] = 0
                else:
                    scores[level]['partial'][type_]['rec'] = scores[level]['partial'][type_]['rec'] / \
                        scores[level]['partial'][type_]['rec_count'] * 1.0
                if scores[level]['partial'][type_]['acc'] == 0 and scores[level]['partial'][type_]['rec'] == 0:
                    scores[level]['partial'][type_]['f1'] = 1
                else:
                    scores[level]['partial'][type_]['f1'] = \
                        2.0 * scores[level]['partial'][type_]['acc'] * scores[level]['partial'][type_]['rec'] / (
                            scores[level]['partial'][type_]['rec'] + scores[level]['partial'][type_]['acc'])

    print_scores(scores, etype)

def eval_exec_match(db, p_str, g_str, pred, gold):
    """
    return 1 if the values between prediction and gold are matching
    in the corresponding index. Currently not support multiple col_unit(pairs).
    """
    conn = sqlite3.connect(db)
    cursor = conn.cursor()
    try:
        cursor.execute(p_str)
        p_res = cursor.fetchall()
    except Exception:
        return False

    cursor.execute(g_str)
    q_res = cursor.fetchall()

    def res_map(res, val_units):
        rmap = {}
        for idx, val_unit in enumerate(val_units):
            key = tuple(val_unit[1]) if not val_unit[2] else (
                val_unit[0], tuple(val_unit[1]), tuple(val_unit[2]))
            rmap[key] = [r[idx] for r in res]
        return rmap

    p_val_units = [unit[1] for unit in pred['select'][1]]
    q_val_units = [unit[1] for unit in gold['select'][1]]
    return res_map(p_res, p_val_units) == res_map(q_res, q_val_units)

# Rebuild SQL functions for value evaluation
def rebuild_cond_unit_val(cond_unit):
    if cond_unit is None or not DISABLE_VALUE:
        return cond_unit

    not_op, op_id, val_unit, val1, val2 = cond_unit
    if type(val1) is not dict:
        val1 = None
    else:
        val1 = rebuild_sql_val(val1)
    if type(val2) is not dict:
        val2 = None
    else:
        val2 = rebuild_sql_val(val2)
    return not_op, op_id, val_unit, val1, val2

def rebuild_condition_val(condition):
    if condition is None or not DISABLE_VALUE:
        return condition

    res = []
    for idx, it in enumerate(condition):
        if idx % 2 == 0:
            res.append(rebuild_cond_unit_val(it))
        else:
            res.append(it)
    return res

def rebuild_sql_val(sql):
    if sql is None or not DISABLE_VALUE:
        return sql

    sql['from']['conds'] = rebuild_condition_val(sql['from']['conds'])
    sql['having'] = rebuild_condition_val(sql['having'])
    sql['where'] = rebuild_condition_val(sql['where'])
    for tab_unit in sql['from']['table_units']:
        if tab_unit[0] == 'sql':
            tab_unit = ('sql', rebuild_sql_val(tab_unit[1]))
    sql['intersect'] = rebuild_sql_val(sql['intersect'])
    sql['except'] = rebuild_sql_val(sql['except'])
    sql['union'] = rebuild_sql_val(sql['union'])

    return sql

# Rebuild SQL functions for foreign key evaluation
def build_valid_col_units(table_units, schema):
    col_names = schema._table['column_names_original']
    tab_ids = [table_unit[1]
               for table_unit in table_units if table_unit[0] == TABLE_TYPE['table_unit']]
    valid_col_units = []
    for value in schema.idMap.values():
        if col_names[value][0] in tab_ids:
            valid_col_units.append(value)
    return valid_col_units

def rebuild_col_unit_col(valid_col_units, col_unit, kmap):
    if col_unit is None:
        return col_unit

    agg_id, col_id, distinct = col_unit
    if col_id in kmap and col_id in valid_col_units:
        col_id = kmap[col_id]
    if DISABLE_DISTINCT:
        distinct = None
    return agg_id, col_id, distinct

def rebuild_val_unit_col(valid_col_units, val_unit, kmap):
    if val_unit is None:
        return val_unit

    unit_op, col_unit1, col_unit2 = val_unit
    col_unit1 = rebuild_col_unit_col(valid_col_units, col_unit1, kmap)
    col_unit2 = rebuild_col_unit_col(valid_col_units, col_unit2, kmap)
    return unit_op, col_unit1, col_unit2

def rebuild_table_unit_col(valid_col_units, table_unit, kmap):
    if table_unit is None:
        return table_unit

    table_type, col_unit_or_sql = table_unit
    if isinstance(col_unit_or_sql, tuple):
        col_unit_or_sql = rebuild_col_unit_col(valid_col_units, col_unit_or_sql, kmap)
    return table_type, col_unit_or_sql

def rebuild_cond_unit_col(valid_col_units, cond_unit, kmap):
    if cond_unit is None:
        return cond_unit

    not_op, op_id, val_unit, val1, val2 = cond_unit
    val_unit = rebuild_val_unit_col(valid_col_units, val_unit, kmap)
    if isinstance(val1, dict):
        val1 = rebuild_sql_col(valid_col_units, val1, kmap)
    if isinstance(val2, dict):
        val2 = rebuild_sql_col(valid_col_units, val2, kmap)
    return not_op, op_id, val_unit, val1, val2

def rebuild_condition_col(valid_col_units, condition, kmap):
    for idx in range(len(condition)):
        if idx % 2 == 0:
            condition[idx] = rebuild_cond_unit_col(valid_col_units, condition[idx], kmap)
    return condition

def rebuild_select_col(valid_col_units, sel, kmap):
    if sel is None:
        return sel
    distinct, _list = sel
    new_list = []
    for it in _list:
        agg_id, val_unit = it
        new_list.append((agg_id, rebuild_val_unit_col(valid_col_units, val_unit, kmap)))
    if DISABLE_DISTINCT:
        distinct = None
    return distinct, new_list

def rebuild_from_col(valid_col_units, from_, kmap):
    if from_ is None:
        return from_

    from_['table_units'] = [rebuild_table_unit_col(valid_col_units, table_unit, kmap) for table_unit in
                            from_['table_units']]
    from_['conds'] = rebuild_condition_col(valid_col_units, from_['conds'], kmap)
    return from_

def rebuild_group_by_col(valid_col_units, group_by, kmap):
    if group_by is None:
        return group_by

    return [rebuild_col_unit_col(valid_col_units, col_unit, kmap) for col_unit in group_by]

def rebuild_order_by_col(valid_col_units, order_by, kmap):
    if order_by is None or len(order_by) == 0:
        return order_by

    direction, val_units = order_by
    new_val_units = [rebuild_val_unit_col(valid_col_units, val_unit, kmap)
                     for val_unit in val_units]
    return direction, new_val_units

def rebuild_sql_col(valid_col_units, sql, kmap):
    if sql is None:
        return sql

    sql['select'] = rebuild_select_col(valid_col_units, sql['select'], kmap)
    sql['from'] = rebuild_from_col(valid_col_units, sql['from'], kmap)
    sql['where'] = rebuild_condition_col(valid_col_units, sql['where'], kmap)
    sql['groupBy'] = rebuild_group_by_col(valid_col_units, sql['groupBy'], kmap)
    sql['orderBy'] = rebuild_order_by_col(valid_col_units, sql['orderBy'], kmap)
    sql['having'] = rebuild_condition_col(valid_col_units, sql['having'], kmap)
    sql['intersect'] = rebuild_sql_col(valid_col_units, sql['intersect'], kmap)
    sql['except'] = rebuild_sql_col(valid_col_units, sql['except'], kmap)
    sql['union'] = rebuild_sql_col(valid_col_units, sql['union'], kmap)

    return sql

def build_foreign_key_map(entry):
    cols_orig = entry["column_names_original"]

    # rebuild cols corresponding to idmap in Schema
    cols = [i for i in range(len(cols_orig))]

    def keyset_in_list(k1, k2, k_list):
        for k_set in k_list:
            if k1 in k_set or k2 in k_set:
                return k_set
        new_k_set = set()
        k_list.append(new_k_set)
        return new_k_set

    foreign_key_list = []
    foreign_keys = entry["foreign_keys"]
    for fkey in foreign_keys:
        key1, key2 = fkey
        key_set = keyset_in_list(key1, key2, foreign_key_list)
        key_set.add(key1)
        key_set.add(key2)

    foreign_key_map = {}
    for key_set in foreign_key_list:
        sorted_list = sorted(list(key_set))
        midx = sorted_list[0]
        for idx in sorted_list:
            foreign_key_map[cols[idx]] = cols[midx]

    return foreign_key_map

def build_foreign_key_map_from_json(table):
    with open(table) as f:
        data = json.load(f)
    tables = {}
    for entry in data:
        tables[entry['db_id']] = build_foreign_key_map(entry)
    return tables

def is_col_valid(col_unit, in_where=False):
    if col_unit is None:
        return True
    if col_unit[0] == 0 and col_unit[1] == 0:
        return False
    if in_where and col_unit[0] != 0:
        return False
    return True

def is_query_valid(_sql, schema):
    select_body = _sql['select'][1]
    if len(select_body) != len(set(select_body)):
        return False

    conds = _sql['from']['conds'][::2]
    tab_in_conds = set()
    for cond in conds:
        tab1 = schema._table['column_names_original'][cond[2][1][1]][0]
        tab2 = schema._table['column_names_original'][cond[3][1]][0]
        if tab1 == -1 or tab2 == -1:
            return False
        tab_in_conds.add(tab1)
        tab_in_conds.add(tab2)

    table_units = _sql['from']['table_units']
    tab_in_from = set()
    for tab in table_units:
        if isinstance(tab[1], int):
            tab_in_from.add(tab[1])

    if len(tab_in_from) > 1 and tab_in_conds != tab_in_from:
        return False

    if len(table_units) == 1 and len(conds) > 0:
        return False

    where_conds = _sql['where'][::2]
    having_conds = _sql['having'][::2]
    for cond in where_conds:
        if isinstance(cond[3], dict) and not is_sql_valid(cond[3], schema):
            return False
        if not is_col_valid(cond[2][1], True):
            return False
        if not is_col_valid(cond[2][2], True):
            return False
    for cond in having_conds:
        if isinstance(cond[3], dict) and not is_sql_valid(cond[3], schema):
            return False
        if not is_col_valid(cond[2][1]):
            return False
        if not is_col_valid(cond[2][2]):
            return False
    groupBy = _sql['groupBy']
    for col_unit in groupBy:
        if not is_col_valid(col_unit):
            return False
    if len(_sql['orderBy']) > 0:
        orderBy = _sql['orderBy'][1]
        for val_unit in orderBy:
            if not is_col_valid(val_unit[1]):
                return False
            if not is_col_valid(val_unit[2]):
                return False
    return True

def is_sql_valid(_sql, schema):
    if _sql['except']:
        if not is_query_valid(_sql['except'], schema):
            return False
    elif _sql['union']:
        if not is_query_valid(_sql['union'], schema):
            return False
    elif _sql['intersect']:
        if not is_query_valid(_sql['intersect'], schema):
            return False

    return is_query_valid(_sql, schema)

# Define new evaluator
@registry.register('evaluator', 'spider')
class SpiderSqlEvaluator:
    def __init__(self, transition_system, args):
        pass

    @staticmethod
    def print_results(results, p_func=print):
        print_scores(results, 'match', p_func=p_func)

    @staticmethod
    def evaluate_dataset(examples, decode_results, out_path, fast_mode=True,
                         test_mode='dev', save_failed_samples=False):
        evaluator = SpiderEvaluator()
        if fast_mode:
            levels = ['easy', 'medium', 'hard', 'extra', 'all']
            partial_types = ['select', 'select(no AGG)', 'where', 'where(no OP)', 'group(no Having)',
                             'group', 'order', 'and/or', 'IUEN', 'keywords']
            etype = 'match'
            scores = {}
            # Init scores
            for level in levels:
                scores[level] = {'count': 0, 'partial': {}, 'exact': 0.}
                for type_ in partial_types:
                    scores[level]['partial'][type_] = {
                        'acc': 0., 'rec': 0., 'f1': 0., 'acc_count': 0, 'rec_count': 0}
            pred_sql = []
            gold_sql = []
            pred_actions = []
            gold_actions = []
            questions = []
            eval_err_num = 0
            idx = 0
            for example, spider_sql in zip(examples, decode_results):
                pruned_hyps = []
                for hyp in spider_sql:
                    if is_sql_valid(hyp.code, example.schema):
                        pruned_hyps.append(hyp)
                gold_spider_sql = example.sql
                _gold_spider_sql = deepcopy(gold_spider_sql)
                pred_spider_sql = pruned_hyps[:1]
                if not pred_spider_sql:
                    # dummy sql
                    surface_sql = 'SELECT *'
                else:
                    surface_sql = SpiderAST(pred_spider_sql[0].code, example.schema._table).get_sql()
                if not pred_spider_sql:
                    p_sql = {
                        "except": None,
                        "from": {
                            "conds": [],
                            "table_units": []
                        },
                        "groupBy": [],
                        "having": [],
                        "intersect": None,
                        "limit": None,
                        "orderBy": [],
                        "select": [
                            False,
                            []
                        ],
                        "union": None,
                        "where": []
                    }
                    pred_spider_sql = SpiderDecodeHypothesis(example.schema)
                    pred_spider_sql.code = p_sql
                    eval_err_num += 1
                else:
                    pred_spider_sql = pred_spider_sql[0]
                _pred_spider_sql = deepcopy(pred_spider_sql.code)

                schema = example.schema
                kmap = build_foreign_key_map(example.schema._table)
                g_valid_col_units = build_valid_col_units(
                    gold_spider_sql['from']['table_units'], schema)
                gold_spider_sql = rebuild_sql_val(gold_spider_sql)
                gold_spider_sql = rebuild_sql_col(g_valid_col_units, gold_spider_sql, kmap)
                p_valid_col_units = build_valid_col_units(
                    pred_spider_sql.code['from']['table_units'], schema)
                pred_spider_sql.code = rebuild_sql_val(pred_spider_sql.code)
                pred_spider_sql.code = rebuild_sql_col(
                    p_valid_col_units, pred_spider_sql.code, kmap)

                hardness = evaluator.eval_hardness(gold_spider_sql)
                scores[hardness]['count'] += 1
                scores['all']['count'] += 1
                exact_score = evaluator.eval_exact_match(pred_spider_sql.code, gold_spider_sql)
                if exact_score == 0 and save_failed_samples:
                    f_out = open(os.path.join(out_path, "%d-%s.md" % (idx, hardness)), "w")
                    f_out.write('### Question\n%s\n' % example.original)
                    f_out.write('\n### Spider SQL\n')
                    f_out.write('- ***pred***: ')
                    f_out.write('%s\n' % surface_sql)
                    f_out.write('- ***gold***: ')
                    f_out.write('%s\n' % example.tgt_code)
                    f_out.write('\n### Action Sequences Diff\n')
                    pred_actions = []
                    for a in pred_spider_sql.actions:
                        pred_actions.append(str(a).replace('*', '\*'))
                    gold_actions = []
                    for a in example.tgt_actions:
                        gold_actions.append(str(a.action).replace('*', '\*'))
                    for line in difflib.unified_diff(pred_actions, gold_actions, fromfile='pred', tofile='gold'):
                        f_out.write('\t%s\n' % line)

                    f_out.write('\n### Schema\n')
                    f_out.write('\tcol_id,\ttab_name,\tcol_name\n')
                    for _id, (tab_id, col_name) in enumerate(example.schema._table['exp_column_names_original']):
                        f_out.write('\t%d,\t%s,\t%s\n' % (
                            _id, example.schema._table['exp_table_names_original'][tab_id], col_name))
                    f_out.write('\n### Primary Keys\n%s\n' %
                                str(example.schema._table['exp_primary_keys']))
                    f_out.close()
                    questions.append(" ".join(example.src_sent))
                    pred_sql.append(_pred_spider_sql)
                    gold_sql.append(_gold_spider_sql)
                    pred_actions.append(pred_spider_sql.actions)
                    gold_actions.append([a.action for a in example.tgt_actions])
                partial_scores = evaluator.partial_scores
                scores[hardness]['exact'] += exact_score
                scores['all']['exact'] += exact_score
                for type_ in partial_types:
                    if partial_scores[type_]['pred_total'] > 0:
                        scores[hardness]['partial'][type_]['acc'] += partial_scores[type_]['acc']
                        scores[hardness]['partial'][type_]['acc_count'] += 1
                    if partial_scores[type_]['label_total'] > 0:
                        scores[hardness]['partial'][type_]['rec'] += partial_scores[type_]['rec']
                        scores[hardness]['partial'][type_]['rec_count'] += 1
                    scores[hardness]['partial'][type_]['f1'] += partial_scores[type_]['f1']
                    if partial_scores[type_]['pred_total'] > 0:
                        scores['all']['partial'][type_]['acc'] += partial_scores[type_]['acc']
                        scores['all']['partial'][type_]['acc_count'] += 1
                    if partial_scores[type_]['label_total'] > 0:
                        scores['all']['partial'][type_]['rec'] += partial_scores[type_]['rec']
                        scores['all']['partial'][type_]['rec_count'] += 1
                    scores['all']['partial'][type_]['f1'] += partial_scores[type_]['f1']
                idx += 1

            for level in levels:
                if scores[level]['count'] == 0:
                    continue

                if etype in ["all", "match"]:
                    scores[level]['exact'] /= scores[level]['count']
                    for type_ in partial_types:
                        if scores[level]['partial'][type_]['acc_count'] == 0:
                            scores[level]['partial'][type_]['acc'] = 0
                        else:
                            scores[level]['partial'][type_]['acc'] = scores[level]['partial'][type_]['acc'] /\
                                scores[level]['partial'][type_]['acc_count'] * 1.0
                        if scores[level]['partial'][type_]['rec_count'] == 0:
                            scores[level]['partial'][type_]['rec'] = 0
                        else:
                            scores[level]['partial'][type_]['rec'] = scores[level]['partial'][type_]['rec'] /\
                                scores[level]['partial'][type_]['rec_count'] * 1.0
                        if scores[level]['partial'][type_]['acc'] == 0 and scores[level]['partial'][type_]['rec'] == 0:
                            scores[level]['partial'][type_]['f1'] = 1
                        else:
                            scores[level]['partial'][type_]['f1'] = \
                                2.0 * scores[level]['partial'][type_]['acc'] * scores[level]['partial'][type_]['rec'] /\
                                (scores[level]['partial'][type_]['rec'] + scores[level]['partial'][type_]['acc'])

            scores["accuracy"] = scores["all"]["exact"]

            out_dict = {
                "questions": questions,
                "pred_sql": pred_sql,
                "gold_sql": gold_sql,
                "pred_actions": pred_actions,
                "gold_actions": gold_actions,
            }
            print("eval_err_num:{}".format(eval_err_num))
            if save_failed_samples:
                with open(os.path.join(out_path, "failed_samples.pkl"), "wb") as out:
                    pickle.dump(out_dict, out)
        else:
            scores = {'accuracy': 0.0}
            for example, spider_sql in zip(examples, decode_results):
                pruned_hyps = []
                for hyp in spider_sql:
                    if is_sql_valid(hyp.code, example.schema):
                        pruned_hyps.append(hyp)
                gold_spider_sql = example.sql
                schema = example.schema
                kmap = build_foreign_key_map(example.schema._table)
                g_valid_col_units = build_valid_col_units(
                    gold_spider_sql['from']['table_units'], schema)
                gold_spider_sql = rebuild_sql_val(gold_spider_sql)
                gold_spider_sql = rebuild_sql_col(g_valid_col_units, gold_spider_sql, kmap)

                flag = False
                for hyp in pruned_hyps:
                    p_valid_col_units = build_valid_col_units(
                        hyp.code['from']['table_units'], schema)
                    hyp.code = rebuild_sql_val(hyp.code)
                    hyp.code = rebuild_sql_col(p_valid_col_units, hyp.code, kmap)
                    exact_score = evaluator.eval_exact_match(hyp.code, gold_spider_sql)
                    if exact_score:
                        flag = True
                        break
                scores['accuracy'] += 1.0 if flag else 0.0
            scores['accuracy'] /= len(examples)

        return scores

@registry.register('evaluator', 'spider-action-evaluator')
def create_spider_action_prediction_evaluator(transition_system, eval_top_pred_only=True, for_inference=False):
    def evaluate_action_predictions(examples, predictions, exp_dir_path):
        """
        @param examples_batches: list(list(example))
        @param predictions_batches: list(list(tensor))
        @param exp_dir_path: str
        @return:
        """
        eva_output_path = None
        if (exp_dir_path is not None) and (not for_inference):
            eva_output_path = os.path.join(exp_dir_path, 'eval_output')
            if not os.path.exists(eva_output_path):
                os.makedirs(eva_output_path)
        logger = logging.getLogger()

        code_results = list()
        for data_idx, pred in enumerate(predictions):
            decoded_hyps = list()
            for hyp_id, hyp in enumerate(pred):
                try:
                    hyp.code = transition_system.ast_to_surface_code(
                        hyp.tree, examples[data_idx].schema
                    )
                    decoded_hyps.append(hyp)
                except Exception:
                    logger.error('Exception in converting tree to code:')
                    logger.error(traceback.format_stack())
                    logger.error(traceback.format_exc())
                    logger.error('-' * 60)
                    logger.error('Example: %s\nIntent: %s\nTarget Code:\n%s\nHypothesis[%d]:\n%s' % (
                        data_idx, ' '.join(examples[data_idx].src_sent),
                        examples[data_idx].tgt_code, hyp_id, hyp.tree.to_string()
                    ))
                    logger.error('-' * 60)
            code_results.append(decoded_hyps)

        if for_inference:
            eval_result = []
            for example, spider_sql in zip(examples, code_results):
                pruned_hyps = []
                for hyp in spider_sql:
                    if is_sql_valid(hyp.code, example.schema):
                        pruned_hyps.append(hyp)
                pred_spider_sql = pruned_hyps[:1]
                if not pred_spider_sql:
                    # dummy sql
                    surface_sql = 'SELECT *'
                else:
                    pred_spider_sql = pred_spider_sql[0].code
                    surface_sql = SpiderAST(pred_spider_sql, example.schema._table).get_sql()
                eval_result.append(surface_sql)
            with open('predicted_sql.txt', 'w') as f:
                for q in eval_result:
                    f.write(q + '\n')
        else:
            evaluator = SpiderSqlEvaluator(None, None)
            eval_results = evaluator.evaluate_dataset(
                examples, code_results, eva_output_path, fast_mode=eval_top_pred_only,
                test_mode='dev', save_failed_samples=eva_output_path is not None)
            print_scores(eval_results, 'match', p_func=logger.info)
            eval_result = eval_results['accuracy']
        return eval_result

    return evaluate_action_predictions

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--gold', dest='gold', type=str)
    parser.add_argument('--pred', dest='pred', type=str)
    parser.add_argument('--db', dest='db', type=str)
    parser.add_argument('--table', dest='table', type=str)
    parser.add_argument('--etype', dest='etype', type=str)
    args = parser.parse_args()

    gold = args.gold
    pred = args.pred
    db_dir = args.db
    table = args.table
    etype = args.etype

    assert etype in ["all", "exec", "match"], "Unknown evaluation method"

    kmaps = build_foreign_key_map_from_json(table)

    evaluate(gold, pred, db_dir, etype, kmaps)


# I think there can be something cool to come from this eventually, I'm bad at picking colors
import json
from json import JSONDecoder

class ColorSchemaDeserializationError(Exception):
    """ Exception deserializing color schema from JSON """
    pass

class ColorSchema:
    def __init__(self, name: str = None, serialized_json: str = None):
        if serialized_json:
            try:
                data = json.loads(serialized_json)
                self._name = data["colorSchema"]["identifier"]
                for each_rgb_tuple in data["colorSchema"]["rgb_values"]:
                    hex_code = "%02x%02x%02x" % (tuple(each_rgb_tuple))
                    self._color_table.append(hex_code)
            except JSONDecoder:
                raise ColorSchemaDeserializationError
            except IndexError:
                raise ColorSchemaDeserializationError
        else:
            self._name = name
            self._color_table = []

    def _get_color_for_index(self, index):
        if index < len(self._color_table):
            return self._color_table[index]
        else:
            corrected_index = index % len(self._color_table)
            return self._color_table[corrected_index]

    def hex_code(self, index: int) -> str:
        return self._get_color_for_index(index)

    def rgb(self, index: int) -> tuple:
        rgb = self._get_color_for_index(index)
        red = int(rgb[0:2], 16)
        green = int(rgb[2:4], 16)
        blue = int(rgb[4:6], 16)
        return red, green, blue

    def __str__(self) -> str:
        if self._name:
            return self._name
        else:
            return "DEFAULT"

    def serialize_json(self) -> str:
        output = {
            "colorSchema": {"identifier": str(self),
                            "rgb_values": []}
        }
        for x in range(len(self._color_table)):
            output["colorSchema"]["rgb_values"].append(self.rgb(x))
        return json.dumps(output)

class DefaultColorSchema(ColorSchema):
    def __init__(self):
        super().__init__(name="DEFAULT")
        colors = ["233142",
                  "facf5a",
                  "ff5959"
                  "4f9da6",
                  "022c43",
                  "ffd700",
                  "115173",
                  "053f5e",
                  "3a9679",
                  "fabc60",
                  "11144c",
                  "085f63",
                  "49beb7",
                  "facf5a",
                  "ff5959"]
        self._color_table.extend(colors)


import scipy.io
import numpy as np

#frames the signal @x into frames of size @frame_size separated by @hop samples
def get_frames( x, frame_size, hop ):
    start_idx = 0
    end_idx = frame_size
    frames = []
    limit = x.shape[1]

    while end_idx <= limit:
        frames.append( x[:, start_idx:end_idx] )
        start_idx = start_idx + hop
        end_idx = start_idx + frame_size

    frames = np.float32(frames)

    return np.swapaxes( frames, 1, 2 )

def long_prediction( model, x, length, timesteps ):
    seed = x
    for i in range( length ):
        y = model.predict( np.expand_dims( seed, axis= 0 ) )
        x = np.vstack( (x, y[0, -1, 0]) )
        seed = x[-timesteps:]

    return x

def prepare_data( data ):
    sample_length = 1
    timesteps = 99
    hop = 1

    data = ( data - data[0].mean() ) / data[0].std()

    X_train = get_frames( data[:, 0:-1], timesteps, hop )
    y_train = get_frames( data[:, 1:], timesteps, hop )

    return ( X_train, y_train )


from django.shortcuts import render,redirect
from django.core.paginator import Paginator
from siteSettings.models import SiteSettings
from django.contrib import messages
from users.models import User
from carts.models import *
from .models import *

def courses_page(request):
    if request.user.is_authenticated:
        if request.user.phone_authentication != True:
            return redirect('account:code_page')
        else:
            user_info = User.objects.filter(username=request.user.username).first()

            if user_info.block == True:
                messages.error(request, 'اکانت شما مسدود شده است لطفا با پشتیبانی تماس بگیرید')
                return redirect('contact:contactus_page')
    else: return redirect('account:login_page')

    posts = Courses.objects.filter(CourseStatus=True).all()
    paginator = Paginator(posts, 9)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    settings = SiteSettings.objects.last()
    title = settings.title + ' ' + '-' + ' ' + 'دوره ها'
    context = {
        'title': title,
        'posts': page_obj,
    }
    return render(request,'courses/courses_page/coursesPage.html',context)

def ShowCourse_page(request,id):
    if request.user.is_authenticated:
        if request.user.phone_authentication != True:
            return redirect('account:code_page')
        else:
            user_info = User.objects.filter(username=request.user.username).first()

            if user_info.block == True:
                messages.error(request, 'اکانت شما مسدود شده است لطفا با پشتیبانی تماس بگیرید')
                return redirect('contact:contactus_page')
    else: return redirect('account:login_page')

    cart = Course.objects.filter(course_id=id,is_paid=True).first()
    status = False
    if cart is not None:
        status = True

    videos = Videos.objects.filter(course_id=id).all()
    videos_count = Videos.objects.filter(course_id=id).count()
    Introduction_Video = IntroductionVideo.objects.filter(course_id=id).last()

    course = Courses.objects.filter(id=id).first()
    settings = SiteSettings.objects.last()
    title = settings.title + ' ' + '-' + ' ' + f'{course.CourseName}'
    context = {
        'title': title,
        'status': status,
        'videos': videos,
        'videos_count': videos_count,
        'keyWord': course.keyWord,
        'Introduction_Video': Introduction_Video,
        'course': course,
    }

    return render(request,'courses/showCourse_page/show_course.html',context)


#!/usr/bin/env python3
with open('numbers.txt', 'r') as f:
    text = f.read().split("\n")[:-1]
enumerated_list = list(enumerate(text, 1))
dictionary = dict(enumerated_list)
for i in dictionary.keys():
    print(f"{hex(hash(dictionary[i]))}: '{dictionary[i]}'")
lengths = []
for i in text:
    lengths.append(len(i))
max = max(lengths)
padded = []
for i in text:
    j = i.rjust(max, '0')
    padded.append(j)
print(padded)


import os
import bs4
from nltk.corpus.reader.api import CorpusReader
from nltk.corpus.reader.api import CategorizedCorpusReader
import nltk
import time
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
from sklearn.cluster import DBSCAN
import numpy as np
from sklearn.metrics import silhouette_samples, silhouette_score
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from sklearn.neighbors import NearestNeighbors
from functools import reduce
from sklearn.feature_extraction import text
from sklearn.decomposition import PCA
from numpy.linalg import svd

CAT_PATTERN = r'([a-z_\s]+)/.*'
DOC_PATTERN = r'(?!\.)[a-z_\s]+/[0-9]+\.html'
TAGS = []
title_TAGS = ['h1']
abstract_TAGS = ['blockquote']

class HTMLCorpusReader(CategorizedCorpusReader, CorpusReader):
    """
    A corpus reader for raw HTML documents to enable preprocessing.
    """

    def __init__(self, root, fileids=DOC_PATTERN, encoding='utf8',
                 tags=TAGS, **kwargs):
        """
        Initialize the corpus reader.  Categorization arguments
        (``cat_pattern``, ``cat_map``, and ``cat_file``) are passed to
        the ``CategorizedCorpusReader`` constructor.  The remaining
        arguments are passed to the ``CorpusReader`` constructor.
        """
        # Add the default category pattern if not passed into the class.
        if not any(key.startswith('cat_') for key in kwargs.keys()):
            kwargs['cat_pattern'] = CAT_PATTERN

        # Initialize the NLTK corpus reader objects
        CategorizedCorpusReader.__init__(self, kwargs)
        CorpusReader.__init__(self, root, fileids, encoding)

        # Save the tags that we specifically want to extract.
        self.tags = tags

    def resolve(self, fileids, categories):
        """
        Returns a list of fileids or categories depending on what is passed
        to each internal corpus reader function. Implemented similarly to
        the NLTK ``CategorizedPlaintextCorpusReader``.
        """
        if fileids is not None and categories is not None:
            raise ValueError("Specify fileids or categories, not both")

        if categories is not None:
            return self.fileids(categories)
        return fileids

    def docs(self, fileids=None, categories=None):
        """
        Returns the complete text of an HTML document, closing the document
        after we are done reading it and yielding it in a memory safe fashion.
        """
        # Resolve the fileids and the categories
        fileids = self.resolve(fileids, categories)

        # Create a generator, loading one document into memory at a time.
        for path in self.abspaths(fileids):
            with open(path, 'r', encoding='UTF-8') as f:
                yield f.read()

    def sizes(self, fileids=None, categories=None):
        """
        Returns a list of tuples, the fileid and size on disk of the file.
        This function is used to detect oddly large files in the corpus.
        """
        # Resolve the fileids and the categories
        fileids = self.resolve(fileids, categories)

        # Create a generator, getting every path and computing filesize
        for path in self.abspaths(fileids):
            yield os.path.getsize(path)

def describe(paragraphs, fileids, categories):
    started = time.time()

    counts = nltk.FreqDist()
    tokens = nltk.FreqDist()

    for para in paragraphs:
        counts['paras'] += 1

        for sent in nltk.sent_tokenize(para):
            counts['sents'] += 1

            for word in nltk.wordpunct_tokenize(sent):
                counts['words'] += 1
                tokens[word] += 1

    n_fileids = len(fileids)
    n_topics = len(categories)

    return {
        'files':  n_fileids,
        'topics': n_topics,
        'paragraphs':  counts['paras'],
        'sentences':  counts['sents'],
        'words':  counts['words'],
        'vocabulary size':  len(tokens),
        'lexical diversity': float(counts['words']) / float(len(tokens)),
        'paragraphs per document':  float(counts['paras']) / float(n_fileids),
        'sentences per paragraph':  float(counts['sents']) / float(counts['paras']),
        'secs':   time.time() - started,
    }

title_corpus = HTMLCorpusReader('', CAT_PATTERN, DOC_PATTERN, tags=title_TAGS)
title_fileids = title_corpus.fileids()
title_documents = title_corpus.docs(categories=title_corpus.categories())
title_htmls = list(title_documents)

abstract_corpus = HTMLCorpusReader('', CAT_PATTERN, DOC_PATTERN, abstract_TAGS)
abstract_fileids = abstract_corpus.fileids()
abstract_documents = abstract_corpus.docs(categories=abstract_corpus.categories())
abstract_htmls = list(abstract_documents)

title_categories = title_corpus.categories()
abstract_categories = abstract_corpus.categories()

def paras(htmls, TAGS): #paragraph로 나누기
    for html in htmls:
        soup = bs4.BeautifulSoup(html, 'lxml')
        for element in soup.find_all(TAGS):
            yield element.text
        soup.decompose()

title_paragraphs = list(paras(title_htmls, title_TAGS))
temp_title_paragraphs = []
for para in title_paragraphs:
    if "Title:" in para: # and len(para)>30
        temp_title_paragraphs.append(para.strip('Title:\n'))
title_paragraphs = temp_title_paragraphs
print("title_paragraphs len: ", len(title_paragraphs))
print("descreibe title_paragraphs", describe(title_paragraphs, title_fileids, title_categories))

abstract_paragraphs = list(paras(abstract_htmls, abstract_TAGS))
print("abstract_paragraphs len: ", len(abstract_paragraphs))
print("descreibe abstract_paragraphs", describe(abstract_paragraphs, abstract_fileids, abstract_categories))

#temp_para = []
#print(abstract_paragraphs[0])
#for para in abstract_paragraphs:
#    temp_para.append(re.sub(r"[^a-zA-Z\s.]", "", para).lower())  # 영문자 + 공백만 남기기)
#abstract_paragraphs = temp_para
#print("descreibe post abstract_paragraphs", describe(abstract_paragraphs, abstract_fileids, abstract_categories))
#print(abstract_paragraphs[0])

papers_list = []
for key, value in zip(title_paragraphs, abstract_paragraphs):
    temp_dict = dict()
    temp_dict['title'] = key
    temp_dict['abstract'] = value
    papers_list.append(temp_dict)

def sklearn_tfidf_vectorize(corpus):
    my_stop_words = text.ENGLISH_STOP_WORDS.union(['abstract', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                                                   'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
                                                   'w', 'x', 'y', 'z'])
    tfidf = TfidfVectorizer(stop_words=my_stop_words)
    return tfidf.fit_transform(corpus)

df = pd.DataFrame(papers_list, columns={'title', 'abstract'})
tf_idf = sklearn_tfidf_vectorize(abstract_paragraphs).todense()
tf_idf_df = pd.DataFrame(tf_idf)
df.to_csv('df.csv')
tf_idf_df.to_csv('tf_idf_df.csv')

def clustering(df_, tf_idf_df_, tf_idf_, eps, min_samples):
    neighbors = NearestNeighbors(n_neighbors=4)
    neighbors_fit = neighbors.fit(tf_idf_df_)
    distances, indices = neighbors_fit.kneighbors(tf_idf_df_)
    distances = np.sort(distances, axis=0)
    distances = distances[:,1]
    plt.plot(distances)
    plt.show()

    if len(eps)*len(min_samples) == 1:
        ncols = 2
    else:
        ncols = len(eps)

    fig, axs = plt.subplots(figsize=(8 * 1, 8), nrows=1, ncols=ncols) #change col
    ind = 0
    result = []
    for i in range(len(eps)):
        print("e,s: ", eps[i], min_samples[i],'\n')
        model = DBSCAN(eps=eps[i], min_samples=min_samples[i])
        clusters = model.fit(tf_idf_df_)
        n_cluster = len(set(clusters.labels_))
        if n_cluster <= 2:
            print("cluster num of", eps[i], min_samples[i], "is 2 or less\n")
            continue
        result.append(model.fit_predict(tf_idf_df_))
        df_['cluster' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])] = result[ind]
        score_samples = silhouette_samples(tf_idf_, df_['cluster' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])])
        df_['silhouette_coeff' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])] = score_samples
        silhouette_s = silhouette_score(tf_idf_, df_['cluster' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])])
        temp = 0
        for p in df_.groupby('cluster' + 'of' + str(eps[i]) + 'and' + str(min_samples[i]))['silhouette_coeff' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])].mean():
            temp += p
        average_score = temp/len(set(clusters.labels_))

        y_lower = 10
        axs[i].set_title(
            'Number of Cluster : ' + str(n_cluster) + '\n' + 'Silhouette Score :' + str(round(silhouette_s, 3)))
        axs[i].set_xlabel("The silhouette coefficient values")
        axs[i].set_ylabel("Cluster label")
        axs[i].set_xlim([-0.1, 1])
        axs[i].set_ylim([0, len(tf_idf_df_) + (n_cluster + 1) * 10])
        axs[i].set_yticks([])  # Clear the yaxis labels / ticks
        axs[i].set_xticks([0, 0.2, 0.4, 0.6, 0.8, 1])

        # 클러스터링 갯수별로 fill_betweenx( )형태의 막대 그래프 표현.
        for j in range(-1, n_cluster-1):
            ith_cluster_sil_values = score_samples[result[ind] == j]
            ith_cluster_sil_values.sort()

            size_cluster_i = ith_cluster_sil_values.shape[0]
            y_upper = y_lower + size_cluster_i

            color = cm.nipy_spectral(float(j) / n_cluster)
            axs[i].fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_sil_values, \
                                   facecolor=color, edgecolor=color, alpha=0.7)
            axs[i].text(-0.05, y_lower + 0.5 * size_cluster_i, str(j))
            y_lower = y_upper + 10

        axs[i].axvline(x=silhouette_s, color="red", linestyle="--")
        ind += 1
    plt.show()

    return result

# svd로 s값 구해서 분산정도 구하고 pca 파라미터 구하기
#u, s, vt = svd(tf_idf)
#s = np.diag(s)
#s_list = []
#for i in range(0, 1000):
#    s_list.append(s[i][i]/np.trace(s))
#
#for i in range(1, 1000):
#    print(1-reduce(lambda a, b: a + b, s_list[:i]))

pca = PCA(n_components=30) # 주성분을 몇개로 할지 결정
principalComponents = pca.fit_transform(tf_idf_df)
principalDf = pd.DataFrame(data=principalComponents)
pca_df = pd.DataFrame(data=principalComponents, index=df.index,
                      columns=[f"pca{num+1}" for num in range(df.shape[1])])

result = pd.DataFrame({'설명가능한 분산 비율(고윳값)':pca.explained_variance_,
             '기여율':pca.explained_variance_ratio_},
            index=np.array([f"pca{num+1}" for num in range(df.shape[1])]))
result['누적기여율'] = result['기여율'].cumsum()
print(result)

eps = [0.05, 0.03, 0.09, 0.05, 0.03, 0.09, 0.05, 0.03, 0.09]
min_samples = [2, 2, 2, 4, 4, 4, 7, 7, 7]
res = clustering(df, principalDf, principalComponents, eps, min_samples)

print(res[0])
print(type(res[0]))

"""
for i, r in enumerate(res):
    if set(r) == None:
        continue
    else:
        for cluster_num in set(r):
            if cluster_num == -1 or cluster_num == 0:
                continue
        # -1,0은 노이즈 판별이 났거나 클러스터링이 안된 경우
            print("cluster num : {}".format(cluster_num))
            temp_df = dff[dff['cluster' + 'of' + str(eps[i]) + 'and' + str(min_samples[i])] == cluster_num] # cluster num 별로 조회
            for title in temp_df['title']:
                print(title) # 제목으로 살펴보자
                print()

    print("-----------------\n")
"""
"""

        print(df.head())
        print("num of clusters: ", len(set(clusters.labels_)))
        print("average_score: " + 'of' + str(e) + 'and' + str(s) + ": ", average_score)
        print("silhouette score: " + 'of' + str(e) + 'and' + str(s) + ": ", silhouette_s)
        print(df.groupby('cluster' + 'of' + str(e) + 'and' + str(s))['silhouette_coeff' + 'of' + str(e) + 'and' + str(s)].mean())
        print("eps, min_samples: " + 'of' + str(e) + 'and' + str(s) + ": ", e, s)"""


"""
This file contains a function to get the coordinates from the GPS
"""
import serial
import time
import string
import pynmea2

#GPS gets coordinates
#UART communication protocols
def getCoordinates():
	port="/dev/ttyAMA0"
	ser=serial.Serial(port, baudrate=9600, timeout=0.5)
	dataout = pynmea2.NMEAStreamReader()
	newdata=ser.readline()

	if newdata[0:6] == "$GPRMC":
		newmsg=pynmea2.parse(newdata)
		lat=newmsg.latitude
		lng=newmsg.longitude
		gps = [lat,lng]
		return gps


numbers = (1,2,3)
# topples are not like arrays, they are not changeable, they are good for making sure a list cannot be changed

# example of unpacking

coordinates = (1, 2, 3)
x, y, z = coordinates
# same as
# x = coordinates[0]
# y = coordinates[1]
# z = coordinates[2]
print(y)


import csv
source = open("Nucleotide.txt", "r")

matrix = [[], [], [], [], [], []]

for x in range(6):
	line = source.readline()
	while True:
		c = source.read(1)
		if c == '>' or c=='':
			break
		if (c!='\n' and c!='\r') :
			matrix[x].append(c)

distance = [[0 for x in range(6)] for x in range(6)]

x=0
for x in range(6):
	for y in range(x+1,6):
		mismatch=0
		for z in range(1679):
			if(matrix[x][z]!=matrix[y][z]):
				mismatch+=1
		distance[x][y]=mismatch/1679.0
		distance[y][x]=mismatch/1679.0

distance[0]=['Cow']+distance[0]
distance[1]=['Sheep']+distance[1]
distance[2]=['Worm']+distance[2]
distance[3]=['Rat']+distance[3]
distance[4]=['Frog']+distance[4]
distance[5]=['Pig']+distance[5]
Poutput = open("Ndistance.txt", "w")
print>>Poutput,"*,Cow,Sheep,Worm,Rat,Frog,Pig"
writer = csv.writer(Poutput)
for i in range(len(distance)):
	writer.writerow(distance[i])
Poutput.close()


'''
main driver file. Responsible for handling user inputs and displaying current GameState
'''

import pygame as p
from PIL import Image
import ChessEngine

p.init()
WIDTH = HEIGHT = 512  # or you can keep it as 512
DIMENSION = 8  # Chess boards are 8x8
SQ_SIZE = HEIGHT // DIMENSION
IMAGES = dict.fromkeys(['wR', 'wN', 'wB', 'wQ', 'wK', 'wp', 'bR', 'bN', 'bB', 'bQ', 'bK', 'bp'])
SET_FPS = 15

def loadImages ():
    for key in list (IMAGES.keys()):  # Use a list instead of a view
        IMAGES[key] = p.transform.scale(p.image.load('/Users/joshmachado/Desktop/Studies/coding/Chess-Engine/images/{}.png'
                                                     .format(key)), (SQ_SIZE, SQ_SIZE))

def main():
    screen = p.display.set_mode((WIDTH, HEIGHT))
    clock = p.time.Clock()
    screen.fill(p.Color('White'))
    loadImages()
    gs = ChessEngine.GameState()
    validMoves = gs.getValidMoves()
    moveMade = False   # flag for when a move is made
    sqSelected = ()    # Empty tuple which will save the position of the last square selected by user
    playerClicks = []  # List of two tuples tracking the clicks eg: [(6,4),(4,4)]

    running = True
    while running:
        for e in p.event.get():
            if e.type == p.QUIT:
                running = False
            elif e.type ==p.MOUSEBUTTONDOWN:
                location = p.mouse.get_pos()
                col = location[0]//SQ_SIZE
                row = location[1]//SQ_SIZE

                if sqSelected == (row, col):  # If the same square is selected twice, it:
                    sqSelected = ()           # Deselects the square
                    playerClicks = []         # Resets playerClicks
                else:
                    sqSelected = (row, col)
                    playerClicks.append(sqSelected)

                if len(playerClicks)==2:    # This indicates that 2 clicks have been made (done after second click)
                    move = ChessEngine.Move(playerClicks[0],playerClicks[1], gs.board)

                    if move in validMoves:
                        print(move.getChessNotation())
                        gs.makeMove(move)
                        moveMade = True
                        sqSelected = ()
                        playerClicks =[]

                    else:
                        playerClicks = [sqSelected]
            # Key handles
            elif e.type == p.KEYDOWN:
                if e.key == p.K_z:      # Undo the move made
                    gs.undoMove()
                    moveMade = True
        if moveMade:
            validMoves = gs.getValidMoves()
            moveMade = False

        drawGameState(screen, gs)
        clock.tick(SET_FPS)
        p.display.flip()

def drawGameState(screen, gs):
    drawBoard(screen)
    drawPieces(screen, gs.board)

'''
drawBoard is going to draw just the board without the pieces.

*Note to self*
Top left square of the chess board is always white irrespective of which colour you're playing with
'''

def drawBoard(screen):
    colours = [p.Color('white'),p.Color('grey')]
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            colour = colours[((r+c)%2)]
            p.draw.rect(screen, colour, p.Rect(c*SQ_SIZE, r*SQ_SIZE, SQ_SIZE, SQ_SIZE))

'''
drawPieces is going to draw the pieces on the board given the current GameState
'''
def drawPieces(screen, board):
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            piece = board[r][c]
            if piece != '--':
                screen.blit(IMAGES[piece], p.Rect(c*SQ_SIZE, r*SQ_SIZE, SQ_SIZE, SQ_SIZE))

if __name__ == '__main__':
    main()


# Creates a basic BLE Advertiser
# Check the Advertiser.py class for more info

import sys
import time
from datetime import datetime, timedelta

from Advertiser import Advertiser

# Advertise
def main():
    dt = datetime.now()
    print("Started at: %s" % dt)

    # Create advertiser
    adv = Advertiser()

    # Initiate BLE stack
    if adv.setupBleStack('Pi Range Tester 2', interval=1000):

        # Start advertising
        adv.enableAdvertisements(en=True)

        # Advertise for n seconds
        time.sleep(600)

        # Disable advertisements
        adv.enableAdvertisements(en=False)

    # Stop
    print("Stopped at: %s" % datetime.now())

# Program start
if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass
    finally:
        print("CTRL-C Detected")
        print("Stopped at: %s" % datetime.now())
        sys.exit()


import numpy as np
import cv2 as cv
import pandas as pd
import random
from math import ceil
from sklearn.utils import shuffle

dt = np.dtype(np.float32)
def generator(batch_size=50):
    while True:
        for j in range(batch_size):
            Xs = []
            Ys = []
            count = 0

            while count < 100:

                day_or_night = random.randint(0,1)

                if day_or_night == 0:
                    folder_day = random.randint(1,13)
                    path_0 = '/../data/archive/Annotations/Annotations/dayTrain/dayClip{}/frameAnnotationsBOX.csv'.format(folder_day)
                    csv_file = pd.read_csv(filepath_or_buffer=path_0, sep=';')

                else:
                    folder_night = random.randint(1,5)
                    path_0 = '/../data/archive/Annotations/Annotations/nightTrain/nightClip{}/frameAnnotationsBOX.csv'.format(folder_night)
                    csv_file = pd.read_csv(filepath_or_buffer=path_0, sep=';')

                # choose picture
                i = random.randint(0, len(csv_file.iloc[:,0].unique())-1) # choose random number of picture in folder
                full_pic_name = csv_file.iloc[:,0].unique()[i]            # with index above choose full name picture
                pic_name = csv_file.iloc[:,0].unique()[i].split('/')[1]   # with index above choose picture

                if day_or_night == 0:
                    path_to_img = '/../data/archive/dayTrain/dayTrain/dayClip{}/frames/'.format(folder_day) + pic_name
                else:
                    path_to_img = '/../data/archive/nightTrain/nightTrain/nightClip{}/frames/'.format(folder_night) + pic_name

                img = cv.imread(path_to_img)
                img = cv.cvtColor(img, cv.COLOR_BGR2RGB)

                # find coordinates
                number_of_same_pic = len(csv_file[csv_file.iloc[:,0] == full_pic_name].iloc[:,0]) # how many pic with same name
                img = cv.copyMakeBorder(img, 200, 200, 200, 200, cv.BORDER_REPLICATE)

                # blobbing
                params = cv.SimpleBlobDetector_Params()
                params.minThreshold = 1
                params.maxThreshold = 255
                params.filterByArea = True
                params.minArea = 100
                params.filterByCircularity = False
                params.filterByConvexity = False
                params.filterByInertia = False

                detector = cv.SimpleBlobDetector_create(params)
                keypoints = detector.detect(img)

                kps = np.array([key for key in keypoints])

                for i in range(number_of_same_pic):
                    if count < 100:
                        # coors of box
                        x1 = csv_file[csv_file.iloc[:,0] == full_pic_name].iloc[i,2]+200
                        y1 = csv_file[csv_file.iloc[:,0] == full_pic_name].iloc[i,3]+200
                        x2 = csv_file[csv_file.iloc[:,0] == full_pic_name].iloc[i,4]+200
                        y2 = csv_file[csv_file.iloc[:,0] == full_pic_name].iloc[i,5]+200

                        # condition for keypoins which are not boxes - TAKES MUCH TIME
                        for key in keypoints:
                            keypoints = [key for key in keypoints if not ((x1-50 < key.pt[0] < x2+50) and (y1-50 < key.pt[1] < y2+50))]

                        random_crop_x1 = random.randint(0, 200-(x2-x1))
                        random_crop_x2 = 200 - random_crop_x1
                        random_crop_y1 = random.randint(0, 200-(y2-y1))
                        random_crop_y2 = 200 - random_crop_y1

                        cropped_img = img[y1-random_crop_y1:y2+random_crop_y2, x1-random_crop_x1:x2+random_crop_x2]

                        new_x1 = random_crop_x1
                        new_y1 = random_crop_y1
                        new_x2 = new_x1 + (x2-x1)
                        new_y2 = new_y1 + (y2-y1)

                        w = cropped_img.shape[1]
                        h = cropped_img.shape[0]

                        Rx = (64 / w)
                        Ry = (64 / h)

                        x1 = ceil(new_x1*Rx)
                        y1 = ceil(new_y1*Ry)
                        x2 = ceil(new_x2*Rx)
                        y2 = ceil(new_y2*Ry)

                        cropped_img = cv.resize(cropped_img, (64, 64))
                        cropped_img = cropped_img.reshape(1, 64, 64, 3)
                        box = np.array([1, x1, y1, x2, y2], dtype=dt)

                        Xs.append(np.array(cropped_img, dtype=dt) / 255.), Ys.append(box)

                        count += 1

                keypoints = keypoints[-5:-1]
                for k in range(len(keypoints)):
                    if count < 100:
                        k_x1 = int(round(keypoints[k].pt[0]-100))
                        k_y1 = int(round(keypoints[k].pt[1]-100))
                        k_x2 = int(round(keypoints[k].pt[0]+100))
                        k_y2 = int(round(keypoints[k].pt[1]+100))

                        cropped_img = img[k_y1:k_y2, k_x1:k_x2]
                        cropped_img = cv.resize(cropped_img, (64, 64))
                        cropped_img = cropped_img.reshape(1, 64, 64, 3)
                        box = np.array([0, 0, 0, 0, 0], dtype=dt)

                        Xs.append(np.array(cropped_img, dtype=dt) / 255.), Ys.append(box)

                        count += 1

            Xs, Ys = shuffle(Xs, Ys)
            yield Xs, Ys


import random
def Rectangle(a,b):
    x1 = a*b
    return x1
def sur(a,b):
    x2 = 2*(a+b)
    return x2

a = input("長辺：")
a = float(a)
b = input("短辺：")
b = float(b)
y=Rectangle(a,b)
z=sur(a,b)
print("面積：",y)
print("周囲：",z)


import matplotlib.pyplot as plt
import numpy as np
from sympy import *
x = Symbol('x')
#! INPUT
print("""
Bắt đầu chương trình
Đây là phương pháp lặp đơn
INPUT
f(x) với f(x)=0
khoảng phân li nghiệm (a,b)

Và thêm SaiSoE đã cho
INPUT
""")

def Phi(x):
# ** là mũ
    # return 1/(x-1.5)
    return -1/3*x**3

a = -2
b = 2
SaiSoE = 10
print(f"""
INPUT
{Phi(x)} , \tvới f(x)=0
khoảng phân li nghiệm (a,b) = ({a},{b})

Và thêm SaiSoE đã cho: SaiSoE = {SaiSoE}
INPUT
""")
#! INPUT
#! KIỂM TRA ĐIỀU KIỆN PP
print(f"""
#! Điều kiện của phương pháp lặp đơn là:
1. Hàm Phi(x) co <=> 0<q<1 (q=MAX_đạo hàm Phi(x))
""")
dao_ham_Phi= diff(Phi(x),x)
print(dao_ham_Phi)
x_ve_hinh=np.linspace(a,b,1000)
y_ve_hinh=[dao_ham_Phi.subs(x,i) for i in x_ve_hinh]
q=max(y_ve_hinh)
print(f"q = {q}\n Xét 0<q<1 => ",end=" ")
if(0<q and q<1):
    print(f"Đúng")
else:
    print(f"Sai")
plt.plot(x_ve_hinh,y_ve_hinh)
plt.show()

#! KIỂM TRA ĐIỀU KIỆN PP

print("Kết thúc chương trình")


"""
INSTANCE METHODS in CLASSES:

In instance methods of a class, the first argument is a reference to the instance of the object itself.

By convention, this argument is named self. It's automatically passed by Python when you call
an instance method on an object.

"""
import sys

class MyClass:
    def instance_method(self) -> None:
        print("This is an instance method using: ", self)

# When you call this method:

obj = MyClass()
obj.instance_method()  # You don't pass 'self', Python does it.
# Here, obj.instance_method() is equivalent to MyClass.instance_method(obj).

sys.exit()


import re

class Rule:
    def __init__(self, regex, fields, pattern):
        self.regex = re.compile(regex)
        self.fields = fields
        self.pattern = pattern

    def apply(self, cls_factory, text):
        return self._setattrs(cls_factory(), self._get_match(text))

    def apply_many(self, cls_factory, text):
        return [
            self._setattrs(cls_factory(), match)
            for match
            in self.regex.finditer(text)
        ]

    def _get_match(self, text):
        match = self.regex.search(text)
        if not match:
            raise ValueError(
                f"Text '{text}' does not match pattern '{self._pattern}'")
        return match

    def _setattrs(self, instance, match):
        items = []
        for field, text in zip(self.fields, match.groups()):
            field.set_value(instance, text)
            items.append(field.get_value(instance))
        instance._items = tuple(items)
        return instance


#!/usr/bin/python

# Import needed to use the GPIO
import RPi.GPIO as GPIO
from time import sleep
import ConfigParser

# Tells the program which numbering scheme you're using for the GPIO pins,
# BCM is the "Broadcom SOC channel", the name of the pins, as opposed
# to the physical numbering of the pins (BOARD).

GPIO.setmode(GPIO.BCM)
config = ConfigParser.SafeConfigParser()
config.read('Config/GPIO.cfg')

pinout_section = 'Pinout'
# Name the GPIO pins
pir_pin = int(config.get(pinout_section, 'pir'))
system_led_pin = int(config.get(pinout_section, 'system_led'))
motion_led_pin = int(config.get(pinout_section, 'motion_led'))
good_button_pin = int(config.get(pinout_section, 'good_button'))
bad_button_pin = int(config.get(pinout_section, 'bad_button'))

# Set the PIR pin as an input and ledPins as outputs
GPIO.setup(pir_pin, GPIO.IN)
GPIO.setup(motion_led_pin, GPIO.OUT)
GPIO.setup(good_button_pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(bad_button_pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(system_led_pin, GPIO.OUT)
GPIO.output(motion_led_pin, False)            # Turn off the green LED
GPIO.output(system_led_pin, True)

# Function to cleanup the GPIO pins
def cleanup():
    GPIO.cleanup()

def check_for_motion():
    # Checks if there's an input from the PIR
    if GPIO.input(pir_pin):
        GPIO.output(motion_led_pin, True)     # Turn on Green LED
        return True
    # If no input detected turns the LED off.
    else:
        GPIO.output(motion_led_pin, False)    # Turn off green LED
        return False


# Imported libraries
import random as r
import time as t
import colorama as col

# Functions

# little helper function that converts an array of numbers into a string
def arr_to_string(arr_nums):
    arr_str = ""
    for n in arr_nums:
        arr_str += str(n) + " "
    return arr_str

# color(number, number, array, boolean)
# slices array from index1(inclusive) - index2(inclusive) from whole_arr.
# If switch = 1, pair is red. If switch = 0, pair is blue
def color(index1, index2, whole_arr, switch):
    num1 = str(whole_arr[index1])
    num2 = str(whole_arr[index2])

    if switch:
        pair = col.Fore.LIGHTRED_EX + num1 + " " + num2
    else:
        pair = col.Fore.LIGHTBLUE_EX + num1 + " " + num2

    if index1 == 0:
        rest_of_arr = whole_arr[2:]
        str_rest = ""
        for n in rest_of_arr:
            str_rest += str(n) + " "
        print(pair + " " + col.Fore.RESET + str_rest)
    else:
        arr_slice_1 = whole_arr[0:index1]
        arr_slice_2 = whole_arr[index2+1:]
        str_slice_1 = ""
        str_slice_2 = ""
        for n in arr_slice_1:
            str_slice_1 += str(n) + " "
        for n in arr_slice_2:
            str_slice_2 += str(n) + " "
        print(str_slice_1 + pair + " " + col.Fore.RESET + str_slice_2)

def bubble_sort(arr_sort, speed):
    input("\nI will now attempt to sort your array. Press \"Enter\" when ready...")
    print()
    arr0 = arr_sort.copy()
    i = 0
    print(arr_to_string(arr0))
    t.sleep(speed)

    # i = how many swaps there aren't. So if i < len(arr0) - 1, it hasn't been fully sorted
    while i < len(arr0) - 1:
        i = 0

        # loops through array in pairs, which is why condition is (n < arr0.length - 1)
        for n in range(0, len(arr0) - 1):
            num1 = arr0[n]
            num2 = arr0[n + 1]

            # if 1st value in the pair is greater than 2nd value, swap
            if num1 - num2 > 0:
                color(n, n + 1, arr0, 1)
                t.sleep(speed)
                arr0[n] = num2
                arr0[n + 1] = num1
                color(n, n+1, arr0, 1)
                t.sleep(speed)
            # if 1st value in pair is not greater than 2nd value, add 1 to i
            else:
                i += 1
                color(n, n+1, arr0, 0)
                t.sleep(speed)

    print('\nFinished! Your newly sorted array is: \n%s' % col.Fore.MAGENTA + str(arr0))
    return arr0

# user generates a random array
# for arr_sort argument in bubble_sort function
def user_interface():
    input('\nWelcome to Array Sort 1.0! Press "Enter" to continue...')
    count = input('\nWe\'re going to generate a random array of numbers.'
                  '\nHow many numbers should be in the array? Please choose between 5 and 25: ')
    num_max = input('\nWhat is the maximum number each number in the array should be? '
                    '\nIn other words, no number will be greater than this number: ')
    num_min = input('\nWhat is the minimum number each number in the array should be?'
                    '\nIn other words, no number will be less than this number: ')
    random_array = [r.randint(int(num_min), int(num_max)) for a in range(0, int(count))]
    print('\nYour array is: \n%s' % col.Fore.MAGENTA + str(random_array) + col.Fore.RESET)
    return random_array

# for speed argument in bubble_sort function
def sorting_speed():
    user_speed = input("\nPlease choose the speed at which you would like to see your array sorted: "
                       "\n1. Slow"
                       "\n2. Normal"
                       "\n3. Instant")
    if user_speed == "1" or user_speed.lower() == "slow":
        return .5
    elif user_speed == "2" or user_speed.lower() == "normal":
        return .3
    elif user_speed == "3" or user_speed.lower() == "instant":
        return 0
    else:
        print("\nI don't understand")
        return sorting_speed()

# Function calls
user_array = user_interface()
speed_choice = sorting_speed()
bubble_sort(user_array, speed_choice)

# End
print(col.Fore.RESET + "\nThank you for using Array Sort 1.0!")


from django.utils.translation import ugettext_lazy as _

from utils.faker_factory import faker

from ..mails import BaseMailView

class ProjectsMemberRoleChangedMailView(BaseMailView):
    """
    """
    template_name = 'mails/projects/member_role_changed.html'

    mandatory_mail_args = [
        'name',
        'user_name',
        'roles',
        'public_url',
    ]
    subject = _('Your role has changed: You are now acting as %(roles)s in %(name)s')
    section = 'projects'

    def get_mock_data(self, optional=True):
        mock_data = {
            'name': '[Project Name]',
            'roles': ['ExO Head Coach', 'Observer'],
            'user_name': '[Name]',
            'disable_notification_url': None,
            'public_url': '/{}'.format(faker.uri_path()),
        }
        return mock_data


import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews

def extract_features(word_list):
    return dict([(word, True) for word in word_list])

if __name__=='__main__':
   # Load positive and negative reviews
   positive_fileids = movie_reviews.fileids('pos')
   negative_fileids = movie_reviews.fileids('neg')
   features_positive = [(extract_features(movie_reviews.words(fileids=[f])),
                         'Positive') for f in positive_fileids]
   features_negative = [(extract_features(movie_reviews.words(fileids=[f])),
                         'Negative') for f in negative_fileids]

   # Split the data into train and test (80/20)
   threshold_factor = 0.8
   # threshold_factor = 0.7
   # threshold_factor = 0.5
   threshold_positive = int(threshold_factor * len(features_positive))
   threshold_negative = int(threshold_factor * len(features_negative))

   features_train = features_positive[:threshold_positive] + features_negative[:threshold_negative]
   features_test = features_positive[threshold_positive:] + features_negative[threshold_negative:]
   print("\nNumber of training datapoints:", len(features_train))
   print("Number of test datapoints:", len(features_test))

   # Train a Naive Bayes classifier
   classifier = NaiveBayesClassifier.train(features_train)
   print("\nAccuracy of the classifier:", nltk.classify.util.accuracy(classifier, features_test))

   print("\nTop 10 most informative words:")
   for item in classifier.most_informative_features()[:10]:
        print(item[0])

    # Sample input reviews
   # input_reviews = [
   #     "It is an amazing movie",
   #     "This is a dull movie. I would never recommend it to anyone.",
   #     "The cinematography is pretty great in this movie",
   #     "The direction was terrible and the story was all over the place"
   # ]
   input_reviews = [
       "This was easy to put together. It is sturdy ad a perfect fit for my daughter's room. We have one drawer that sticks a little COMMA  but it works.",
       "I loved the look of this dresser in the store and decided to take the plunge. It was a major project to assemble (6-ish hours for one relatively handy person without power tools) COMMA  but the finished product looks great and stores a ton of clothes! The directions could definitely be a little clearer on assembling the middle divider pieces COMMA  which looks wrong even when done correctly and the dimples in the wood for orientation look like holes in the instructions. I couldn't get two of the four screws that connect the front face to the dresser top to go in (screws too short or holes not quite aligned) COMMA  but thankfully there were many other points of attachment and it's not at all obvious that they're missing. And mine came with metal (not plastic) cam locks COMMA  which is a good thing. Great buy!",
       "We were very disappointed to realize that the hemnes set in white is made of mostly particle board. We were under the impression that the all the hemnes line was made of solid wood. After further investigation it seems as though all the dressers are made of wood except the white ones. Not sure why this is and is very misleading",
       "I not only purchased the dresser but I bought the matching chest I'd drawers. The pieces took a while to put together but they are worth the time. Great product."
   ]
   print("\nPredictions:")
   for review in input_reviews:
       print("\nReview:", review)
       probdist = classifier.prob_classify(extract_features(review.split()))
       pred_sentiment = probdist.max()
       print("Predicted sentiment:", pred_sentiment)
       print("Probability:", round(probdist.prob(pred_sentiment), 2))