import pandas
from matplotlib import pyplot as plt
from tensorflow import keras
import numpy as np
import seaborn as sb

def _get_output(layer, X, model_input):
    """Get output of partial model"""
    partial_model = keras.models.Model(inputs=model_input,
                                       outputs=layer.output)
    y = partial_model.predict(X)
    return y

def array_to_summary(x, fmt='boxplot'):
    if fmt == 'boxplot':
        y = {'med': np.median(x),
             'q1': np.percentile(x, 25),
             'q3': np.percentile(x, 75),
             'whislo': min(x),
             'whishi': max(x)
             }
    elif fmt == 'histogram':
        # Power of 2 bins covering datasets range
        high = np.ceil(np.log2(abs(max(x)))) + 1
        low = np.floor(np.log2(abs(min(x)))) - 1
        bits = np.arange(low, high, 1)
        bins = 2 ** bits
        h, b = np.histogram(x, bins=bins)
        h = h * 1. / float(sum(h))  # normalize
        y = {'h': h,
             'b': np.log2(b)}
    return y

def histogram(data, fmt='longform'):
    f = plt.figure()
    from matplotlib.ticker import MaxNLocator
    n = len(data) if fmt == 'summary' else len(data['weight'].unique())
    colors = sb.color_palette("husl", n)
    if fmt == 'longform':
        for i, weight in enumerate(data['weight'].unique()):
            y = array_to_summary(data[data['weight'] == weight]['x'], fmt='histogram')
            plt.bar(y['b'][:-1], y['h'], width=1, fill=False, label=weight, edgecolor=colors[i])
    elif fmt == 'summary':
        for i, weight in enumerate(data):
            plt.bar(weight['b'][:-1], weight['h'], width=1, fill=False, label=weight['weight'], edgecolor=colors[i])

    plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
    plt.xlabel('log2(x)')
    plt.ylabel('frequency')
    plt.legend()
    return f

def boxplot(data, fmt='longform'):
    if fmt == 'longform':
        f = plt.figure()  # figsize=(3, 3))
        hue = 'layer' if 'layer' in data.keys() else None
        vp = sb.boxplot(x='x', y='weight', hue=hue, data=data[data['x'] > 0], showfliers=False)
        vp.set_yticklabels(vp.get_yticklabels(), rotation=45, ha='right')
        if hue is not None:
            vp.get_legend().remove()
        vp.set_xscale('log', base=2)
        return f
    elif fmt == 'summary':
        from matplotlib.patches import Rectangle
        medianprops = dict(linestyle='-', color='k')
        f, ax = plt.subplots(1, 1)
        data.reverse()
        colors = sb.color_palette("Blues", len(data))
        bp = ax.bxp(data, showfliers=False, vert=False, medianprops=medianprops)
        # add colored boxes
        for line, color in zip(bp['boxes'], colors):
            x = line.get_xdata()
            xl, xh = min(x), max(x)
            y = line.get_ydata()
            yl, yh = min(y), max(y)
            rect = Rectangle((xl, yl), (xh - xl), (yh - yl), fill=True, color=color)
            ax.add_patch(rect)
        ax.set_yticklabels([d['weight'] for d in data])
        ax.set_xscale('log', base=2)
        plt.xlabel('x')
        return f
    else:
        return None

def activations_keras_sequence_ts(model, X, fmt='longform', plot='boxplot'):
    # test layer by layer on datasets

    seq_out = []

    for layer in model.layers:
        print("   {}".format(layer.name))
        if not isinstance(layer, keras.layers.InputLayer) and not isinstance(layer, keras.layers.Dropout):
            if isinstance(layer, keras.layers.RNN):
                layer.return_sequences = True
                # layer.return_state = True
            y = _get_output(layer, X, model.input)
            if fmt == 'longform':
                # return long form pandas dataframe for
                # seaborn boxplot
                data = {'value': [], 'layer_ts': []}
            elif fmt == 'summary':
                # return summary statistics for matplotlib.axes.Axes.bxp
                # or histogram bin edges and heights
                data = []
            for i in range(0, y.shape[1]):
                if fmt == 'longform':
                    data['value'].extend(y[:, i, :].flatten().tolist())
                    data['layer_ts'].extend([layer.name + '_ts_' + str(i) for _ in range(len(y[:, i, :].flatten()))])
                elif fmt == 'summary':
                    data.append(array_to_summary(y[:, i, :].flatten(), fmt=plot))
                    data[-1]['layer_name'] = layer.name
            if fmt == 'longform':
                seq_out.append(pandas.DataFrame(data))

    return seq_out


def solution(p):
    answer = ''
    def inverse(s):
        result = ''
        for i in s:
            if i == '(':
                result += ')'
            else:
                result += '('
        return result
    def correct(s):
        chk = 0
        crt = True
        for i in range(len(s)):
            if s[i] == '(':
                chk += 1
            else:
                chk -= 1
            if chk < 0:
                crt = False
            if chk == 0:
                return (i, crt)
    results = []
    stack = []
    s = p
    while len(s):
        k, check = correct(s)
        if check:
            results.append(s[:k+1])
            s = s[k+1:]
        else:
            stack.append(inverse(s[1:k]))
            stack.append(')')
            results.append('(')
            s = s[k+1:]
    for i in results:
        answer += i
    while stack:
        q = stack.pop()
        answer += q
    return answer
solution("()))((()")

'''
"(()())()"	"(()())()"
")("	"()"
"()))((()"	"()(())()"
'''


import numpy as np
import pandas as pd
from sklearn.datasets import load_digits

def make_dataset():
    """Make the binary digits dataset."""
    digits = load_digits(n_class=2)
    df = pd.DataFrame(np.column_stack((digits.data, digits.target)))
    df = df.rename(columns={64: 'Class'})
    df.loc[df.loc[:, 'Class'] == 0, 'Class'] = -1
    df.to_csv('digits_binary.csv', index=False)

if __name__ == "__main__":
    make_dataset()


from django.urls import include, path
from rest_framework.routers import DefaultRouter

from payment.views import (
    CheckoutAPIView,
    PaymentViewSet,
    StripeCheckoutSessionCreateAPIView,
    StripeWebhookAPIView,
)

app_name = "payment"

router = DefaultRouter()
router.register(r"", PaymentViewSet)

urlpatterns = [
    path("", include(router.urls)),
    path(
        "stripe/create-checkout-session/<int:order_id>/",
        StripeCheckoutSessionCreateAPIView.as_view(),
        name="checkout_session",
    ),
    path("stripe/webhook/", StripeWebhookAPIView.as_view(), name="stripe_webhook"),
    path("checkout/<int:pk>/", CheckoutAPIView.as_view(), name="checkout"),
]


class Solution:
    def respace(self, dictionary: List[str], sentence: str) -> int:
        if not sentence:
            return 0
        d = set(dictionary)
        n = len(sentence)
        dp = [i for i in range(n+1)]
        for i in range(1,n+1):
            for j in range(i):
                if sentence[j:i] in d:
                    dp[i] = min(dp[i], dp[j])
                else:
                    dp[i] = min(dp[i], dp[i-1]+1)
        return dp[-1]


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

# start at root
# use recursion
# check if a subtree contains a 1
# if it contains a 1, do nothing
# if it is only 0s/None, change that child to None
def pruneTree(root):
    # if self.subtree_contains_one(root):
    #     return root
    # else:
    #     return None
    return root if subtree_contains_one(root) else None

def subtree_contains_one(node):
    # base case: leaf
    if node is None:
        return False

    left_has_one = subtree_contains_one(node.left)
    right_has_one = subtree_contains_one(node.right)

    if not left_has_one:
        node.left = None
    if not right_has_one:
        node.right = None

    return left_has_one or right_has_one or (node.val == 1)

root = TreeNode(1)
a = TreeNode(0)
b = TreeNode(0)
c = TreeNode(0)

d = TreeNode(1)
e = TreeNode(0)
f = TreeNode(1)
root.left = a
root.right = d
a.left = b
a.right = c
d.left = e
d.right = f

print(pruneTree(root))


import argparse

import  base64
from models.experimental import *
from utils.datasets import *
from utils.general import *
from utils import torch_utils
import sys

class Smoke_File_Detector():
    def __init__(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--weights', nargs='+', type=str, default=r'Smoke_Fire_Detection-main\weights\smoke.pt', help='model.pt path(s)')
        parser.add_argument('--source', type=str, default='inference/images', help='source')  # file/folder, 0 for webcam
        parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')
        parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold')
        parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
        parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
        parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')
        parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
        parser.add_argument('--augment', action='store_true', help='augmented inference')
        parser.add_argument('--img_name', type=str)
        self.opt = parser.parse_args()

        self.device = torch_utils.select_device(self.opt.device)

        # Initialize
        self.half = self.device.type != 'cpu'  # half precision only supported on CUDA

        # Load models
        self.model = attempt_load(self.opt.weights, map_location=self.device)
        self.imgsz = check_img_size(self.opt.img_size, s=self.model.stride.max())
        if self.half:
            self.model.half()

    # 本地调用
    def detect_test(self,test_list):
        for i,img in enumerate(test_list):
            im0 = img
            img = letterbox(img, new_shape=self.opt.img_size)[0]
            img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416
            img = np.ascontiguousarray(img)  # faster

            # Run inference
            img = torch.from_numpy(img).to(self.device)
            img = img.half() if self.half else img.float()  # uint8 to fp16/32
            img /= 255.0  # 0 - 255 to 0.0 - 1.0
            if img.ndimension() == 3:
                img = img.unsqueeze(0)
            if i == 0:
                batch_img = img
            else:
                batch_img = torch.cat([batch_img,img],axis = 0)

        pred = self.model(batch_img, augment=self.opt.augment)[0]

        # Apply NMS
        pred = non_max_suppression(pred, self.opt.conf_thres, self.opt.iou_thres, classes=self.opt.classes, agnostic=self.opt.agnostic_nms)

        # Process detections
        names = self.model.module.names if hasattr(self.model, 'module') else self.model.names
        batch_results = []
        for i, det in enumerate(pred):  # detections per image
            results = []
            if det is not None and len(det):
                # Rescale boxes from img_size to im0 size
                det[:, :4] = scale_coords(batch_img.shape[2:], det[:, :4], im0.shape).round()
                device = ('cuda' if torch.cuda.is_available() else 'cpu')
                det = det.to(device).data.cpu().numpy()
                for *xyxy, conf, cls in det:
                    w = xyxy[2]-xyxy[0]
                    h = xyxy[3]-xyxy[1]
                    result = {'bbox':xyxy, 'label':names[int(cls)], 'conf':conf}
                    results.append(result)

            batch_results.append(results)

        # print(batch_results)
        return batch_results

    # server调用
    def detect(self, **kwargs):
        params = kwargs
        test_list = [params["img"]]
        for i,img in enumerate(test_list):
            im0 = img
            img = letterbox(img, new_shape=self.opt.img_size)[0]
            img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416
            img = np.ascontiguousarray(img)

            # Run inference
            img = torch.from_numpy(img).to(self.device)
            img = img.half() if self.half else img.float()  # uint8 to fp16/32
            img /= 255.0  # 0 - 255 to 0.0 - 1.0
            if img.ndimension() == 3:
                img = img.unsqueeze(0)
            if i == 0:
                batch_img = img
            else:
                batch_img = torch.cat([batch_img,img],axis = 0)

        pred = self.model(batch_img, augment=self.opt.augment)[0]

        # Apply NMS
        pred = non_max_suppression(pred, self.opt.conf_thres, self.opt.iou_thres, classes=self.opt.classes, agnostic=self.opt.agnostic_nms)

        # Process detections
        names = self.model.module.names if hasattr(self.model, 'module') else self.model.names
        batch_results = []
        for i, det in enumerate(pred):  # detections per image
            results = []
            if det is not None and len(det):
                # Rescale boxes from img_size to im0 size
                det[:, :4] = scale_coords(batch_img.shape[2:], det[:, :4], im0.shape).round()
                device = ('cuda' if torch.cuda.is_available() else 'cpu')
                det = det.to(device).data.cpu().numpy()
                for *xyxy, conf, cls in det:
                    w = xyxy[2]-xyxy[0]
                    h = xyxy[3]-xyxy[1]
                    result = {'bbox':xyxy, 'label':names[int(cls)], 'conf':conf}
                    results.append(result)

            batch_results.append(results)

        # print(batch_results)
        return batch_results

    # 返回框好的图片
    def solveFireImg(self, img, img_name):
        ls = self.detect_test([img])[0]
        # print(ls)
        flag = False
        for dic in ls:
            flag = True
            if(dic['label'] == 'fire'):
                rec = dic['bbox']
                cv2.rectangle(img, (int(rec[0]), int(rec[1])), (int(rec[2]), int(rec[3])), (0, 255, 0), 3)
            if(dic['label'] == 'smoke'):
                rec = dic['bbox']
                cv2.rectangle(img, (int(rec[0]), int(rec[1])), (int(rec[2]), int(rec[3])), (0, 0, 255), 3)
        if(flag) : #有火焰，保存图片
            cv2.imwrite(img_name, img) # 使用传入的参数自动覆盖原来的图片
        return img

if __name__ == "__main__":
    import cv2
    img_name = sys.argv[2]
    img = cv2.imread(img_name)

    det = Smoke_File_Detector()
    arr = det.detect_test([img])[0]
    print(arr)
    # content = det.solveFireImg(img, img_name)
    # # cv2.imshow('test', content)
    # # cv2.waitKey(0)
    # # 保存图片
    # t = time.time()
    # # name = int(round(t * 1000))
    # # img_name='F:\\img\\'+str(name)+'.jpg'
    # img1 = cv2.imencode('.jpg', content)[1]
    # back_2 = base64.b64encode(img1)
    # back_3 = str(back_2, encoding="utf-8")
    # print(back_3, end="")

    """以下的内容一定要注释掉"""
    #测试保存的明亮航参数保存图片
    # img_name = sys.argv[2]
    # print(type(back_2))
    # print("图片")
    # print(img_name)
    # cv2.imwrite(img_name, content)

    # 测试如果直接输出会怎么样


# from re import I
# from socket import if_indextoname
import numpy as np
from copy import deepcopy
import random
from random import choice
from dispatch_alg import LD
from math import radians, sin, atan2,cos,acos
from config import *
import math
import pickle
import osmnx as ox
from tqdm import tqdm
import pandas as pd
import sys
from collections import Counter
import pymongo
import time
import scipy.stats as st
from scipy.stats import skewnorm
"""
Here, we load the information of graph network from graphml file.
"""
G = ox.load_graphml('./input/graph.graphml')
gdf_nodes, gdf_edges = ox.graph_to_gdfs(G)
lat_list = gdf_nodes['y'].tolist()
lng_list = gdf_nodes['x'].tolist()
node_id = gdf_nodes.index.tolist()
node_id_to_lat_lng = {}
node_coord_to_id = {}
for i in range(len(lat_list)):
    node_id_to_lat_lng[node_id[i]] = (lat_list[i], lng_list[i])
    node_coord_to_id[(lng_list[i], lat_list[i])] = node_id[i]

center = (
(env_params['east_lng'] + env_params['west_lng']) / 2, (env_params['north_lat'] + env_params['south_lat']) / 2)
radius = max(abs(env_params['east_lng'] - env_params['west_lng']) / 2,
             abs(env_params['north_lat'] - env_params['south_lat']) / 2)
side = env_params['side']
interval = 2 * radius / side

"""
Here, we build the connection to mongodb, which will be used to speed up access to road network information.
"""
myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["route_network"]

mycollect = mydb['route_list']

# define the function to get zone_id of segment node
def get_zone(lat, lng):
    """
    :param lat: the latitude of coordinate
    :type : float
    :param lng: the longitude of coordinate
    :type lng: float
    :return: the id of zone that the point belongs to
    :rtype: float
    """
    if lat < center[1]:
        i = math.floor(side / 2) - math.ceil((center[1] - lat) / interval) + side % 2
    else:
        i = math.floor(side / 2) + math.ceil((lat - center[1]) / interval) - 1

    if lng < center[0]:
        j = math.floor(side / 2) - math.ceil((center[0] - lng) / interval) + side % 2
    else:
        j = math.floor(side / 2) + math.ceil((lng - center[0]) / interval) - 1
    return i * side + j

result = pd.DataFrame()
nodelist = []
result['lat'] = lat_list
result['lng'] = lng_list
result['node_id'] = gdf_nodes.index.tolist()
for i in range(len(result)):
    nodelist.append(get_zone(lat_list[i], lng_list[i]))
result['grid_id'] = nodelist

"""Generate the available directions for each grid"""
df_available_directions = pd.DataFrame(columns=['zone_id','direction_0','direction_1','direction_2','direction_3','direction_4'])
df_neighbor_centroid = pd.DataFrame()
direction_1_list = []   # up
direction_2_list = []   # down
direction_3_list = []   # left
direction_4_list = []   # right

direction0_available_list = [i for i in range(side**2)] # stay
direction1_available_list = [] # up
direction2_available_list = [] # down
direction3_available_list = [] # left
direction4_available_list = [] # right
centroid_lng_list = []
centroid_lat_list = []

if env_params['rl_mode'] == "matching":
    for i in range(side**2):

        centroid_lng_list.append(result[result['grid_id']==i]['lng'])
        centroid_lat_list.append(result[result['grid_id']==i]['lat'])
        # up
        if math.floor(i / side) == 0:
            direction_1_list.append(0)
            direction1_available_list.append(np.nan)
        else:
            direction_1_list.append(1)
            direction1_available_list.append(i-side)

        # down
        if math.floor(i / side) == side - 1:
            direction_2_list.append(0)
            direction2_available_list.append(np.nan)
        else:
            direction_2_list.append(1)
            direction2_available_list.append(i + side)

        # left
        if i % side == 0:
            direction_3_list.append(0)
            direction3_available_list.append(np.nan)
        else:
            direction_3_list.append(1)
            direction3_available_list.append(i-1)

        # right
        if i % side == side -1:
            direction_4_list.append(0)
            direction4_available_list.append(np.nan)
        else:
            direction_4_list.append(1)
            direction4_available_list.append(i+1)
elif env_params['rl_mode'] == "reposition":
    for i in range(side**2):
        if len(result[result['grid_id']==i]['lng'].values.tolist()) == 0:
            centroid_lng_list.append(lng_list[0])
            centroid_lat_list.append(lat_list[0])
        else:
            centroid_lng_list.append(result[result['grid_id']==i]['lng'].values.tolist()[0])
            centroid_lat_list.append(result[result['grid_id']==i]['lat'].values.tolist()[0])
        # up
        if math.floor(i / side) == 0 and len(result[result['grid_id']==i-side]['lng'].values.tolist()) == 0:
            direction_1_list.append(0)
            direction1_available_list.append(i)
        else:
            direction_1_list.append(1)
            direction1_available_list.append(i-side)

        # down
        if math.floor(i / side) == side - 1 and len(result[result['grid_id']==i+side]['lng'].values.tolist()) == 0:
            direction_2_list.append(0)
            direction2_available_list.append(i)
        else:
            direction_2_list.append(1)
            direction2_available_list.append(i + side)

        # left
        if i % side == 0 and len(result[result['grid_id']==i-1]['lng'].values.tolist()) == 0:
            direction_3_list.append(0)
            direction3_available_list.append(i)
        else:
            direction_3_list.append(1)
            direction3_available_list.append(i-1)

        # right
        if i % side == side -1 and len(result[result['grid_id']==i+1]['lng'].values.tolist()) == 0:
            direction_4_list.append(0)
            direction4_available_list.append(i)
        else:
            direction_4_list.append(1)
            direction4_available_list.append(i+1)

df_available_directions['zone_id'] = [i for i in range(side**2)]
df_available_directions['direction_0'] = 1
df_available_directions['direction_1'] = direction_1_list
df_available_directions['direction_2'] = direction_2_list
df_available_directions['direction_3'] = direction_3_list
df_available_directions['direction_4'] = direction_4_list

df_neighbor_centroid['zone_id'] = direction0_available_list
df_neighbor_centroid['centroid_lng'] = centroid_lng_list
df_neighbor_centroid['centroid_lat'] = centroid_lat_list
df_neighbor_centroid['stay'] = direction0_available_list
df_neighbor_centroid['up'] = direction1_available_list
df_neighbor_centroid['right'] = direction4_available_list
df_neighbor_centroid['down'] = direction2_available_list
df_neighbor_centroid['left'] = direction3_available_list

# rl for matching
def get_exponential_epsilons(initial_epsilon, final_epsilon, steps, decay=0.99, pre_steps=10):
    """
    obtain exponential decay epsilons
    :param initial_epsilon:
    :param final_epsilon:
    :param steps:
    :param decay: decay rate
    :param pre_steps: first several epsilons does note decay
    :return:
    """
    epsilons = []

    # pre randomness
    for i in range(0, pre_steps):
        epsilons.append(deepcopy(initial_epsilon))

    # decay randomness
    epsilon = initial_epsilon
    for i in range(pre_steps, steps):
        epsilon = max(final_epsilon, epsilon * decay)
        epsilons.append(deepcopy(epsilon))

    return np.array(epsilons)
# rl for matching

# rl for repositioning
def s2e(n, total_len = 14):
    n = n.astype(int)
    k = (((n[:,None] & (1 << np.arange(total_len))[::-1])) > 0).astype(np.float64)
    return k
# rl for repositioning

# rl for repositioning
def get_exponential_epsilons(initial_epsilon, final_epsilon, steps, decay=0.99, pre_steps=10):
    """
    obtain exponential decay epsilons
    :param initial_epsilon:
    :param final_epsilon:
    :param steps:
    :param decay: decay rate
    :param pre_steps: first several epsilons does note decay
    :return:
    """
    epsilons = []

    # pre randomness
    for i in range(0, pre_steps):
        epsilons.append(deepcopy(initial_epsilon))

    # decay randomness
    epsilon = initial_epsilon
    for i in range(pre_steps, steps):
        epsilon = max(final_epsilon, epsilon * decay)
        epsilons.append(deepcopy(epsilon))

    return np.array(epsilons)
# rl for repositioning

def distance(coord_1, coord_2):
    """
    :param coord_1: the coordinate of one point
    :type coord_1: tuple -- (latitude,longitude)
    :param coord_2: the coordinate of another point
    :type coord_2: tuple -- (latitude,longitude)
    :return: the manhattan distance between these two points
    :rtype: float
    """
    manhattan_dis = 0
    try:
        lon1,lat1 = coord_1
        lon2,lat2  = coord_2
        lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
        r = 6371
        lat_dis = r * acos(min(1.0, cos(lat1) ** 2 * cos(lon1 - lon2) + sin(lat1) ** 2))
        lon_dis = r * (lat2 - lat1)
        manhattan_dis = (abs(lat_dis) ** 2 + abs(lon_dis) ** 2) ** 0.5
    except Exception as e:
        print(e)
        print(coord_1)
        print(coord_2)
        print(lon1 - lon2)
        print(cos(lat1) ** 2 * cos(lon1 - lon2) + sin(lat1) ** 2)
        print(acos(cos(lat1) ** 2 * cos(lon1 - lon2) + sin(lat1) ** 2))

    return manhattan_dis

def distance_array(coord_1, coord_2):
    """
    :param coord_1: array of coordinate
    :type coord_1: numpy.array
    :param coord_2: array of coordinate
    :type coord_2: numpy.array
    :return: the array of manhattan distance of these two-point pair
    :rtype: numpy.array
    """
    # manhattan_dis = list()
    # for i in range(len(coord_1)):
    #     lon1,lat1 = coord_1[i]
    #     lon2,lat2 = coord_2[i]
    #     lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
    #     r = 6371
    #     lat_dis = r * acos(min(1.0, cos(lat1) ** 2 * cos(lon1 - lon2) + sin(lat1) ** 2))
    #     lon_dis = r * (lat2 - lat1)
    #     manhattan_dis.append((abs(lat_dis) ** 2 + abs(lon_dis) ** 2) ** 0.5)
    # return np.array(manhattan_dis)
    coord_1 = np.array(coord_1).astype(float)
    coord_2 = np.array(coord_2).astype(float)
    coord_1_array = np.radians(coord_1)
    coord_2_array = np.radians(coord_2)
    dlon = coord_2_array[:, 0] - coord_1_array[:, 0]
    dlat = coord_2_array[:, 1] - coord_1_array[:, 1]
    a = np.sin(dlat / 2) ** 2 + np.cos(coord_1_array[:, 1]) * np.cos(coord_2_array[:, 1]) * np.sin(dlon / 2) ** 2
    c = 2 * np.arcsin(a ** 0.5)
    r = 6371
    distance = c * r
    return distance

def get_distance_array(origin_coord_array, dest_coord_array):
    """
    :param origin_coord_array: list of coordinates
    :type origin_coord_array:  list
    :param dest_coord_array:  list of coordinates
    :type dest_coord_array:  list
    :return: tuple like (
    :rtype: list
    """
    dis_array = []
    for i in range(len(origin_coord_array)):
        dis = distance(origin_coord_array[i], dest_coord_array[i])
        dis_array.append(dis)
    dis_array = np.array(dis_array)
    return dis_array

def route_generation_array(origin_coord_array, dest_coord_array, mode='rg'):
    """

    :param origin_coord_array: the K*2 type list, the first column is lng, the second column
                                is lat.
    :type origin_coord_array: numpy.array
    :param dest_coord_array: the K*2 type list, the first column is lng, the second column
                                is lat.
    :type dest_coord_array: numpy.array
    :param mode: the mode of generation; if the value of mode is complete, return the last node of route;
                 if the value of mode is drop_end, the last node of route will be dropped.
    :type mode: string
    :return: tuple like (itinerary_node_list, itinerary_segment_dis_list, dis_array)
             itinerary_node_list contains the id of nodes, itinerary_segment_dis_list contains
             the distance between two nodes, dis_array contains the distance from origin node to
             destination node
    :rtype: tuple
    """
    # print("route generation start")
    # origin_coord_list为 Kx2 的array，第一列为lng，第二列为lat；dest_coord_array同理
    # itinerary_node_list的每一项为一个list，包含了对应路线中的各个节点编号
    # itinerary_segment_dis_list的每一项为一个array，包含了对应路线中的各节点到相邻下一节点的距离
    # dis_array包含了各行程的总里程
    origin_node_list = get_nodeId_from_coordinate(origin_coord_array[:, 0], origin_coord_array[:, 1])
    dest_node_list = get_nodeId_from_coordinate(dest_coord_array[:, 0], dest_coord_array[:, 1])
    itinerary_node_list = []
    itinerary_segment_dis_list = []
    dis_array = []
    if mode == 'ma':
        for origin, dest in zip(origin_node_list, dest_node_list):
            itinerary_node_list.append([dest])
            dis = distance(node_id_to_lat_lng[origin], node_id_to_lat_lng[dest])
            itinerary_segment_dis_list.append([dis])
            dis_array.append(dis)
        return itinerary_node_list, itinerary_segment_dis_list, np.array(dis_array)

    if mode == 'rg':
        # 返回完整itinerary
        for origin,dest in zip(origin_node_list,dest_node_list):
            data = {
                'node': str(origin) + str(dest)
            }
            re = mycollect.find_one(data)
            if re:
                ite = [int(item) for item in re['itinerary_node_list'].strip('[').strip(']').split(', ')]
            else:
                ite = ox.distance.shortest_path(G, origin, dest, weight='length', cpus=16)
            if ite is not None and len(ite) > 1:
                itinerary_node_list.append(ite)
            else:
                itinerary_node_list.append([origin,dest])
        # itinerary_node_list = ox.distance.shortest_path(G, origin_node_list, dest_node_list, weight='length', cpus=16)
        for itinerary_node in itinerary_node_list:

            if itinerary_node is not None:
                itinerary_segment_dis = []
                for i in range(len(itinerary_node) - 1):
                    # dis = nx.shortest_path_length(G, node_id_to_lat_lng[itinerary_node[i]], node_id_to_lat_lng[itinerary_node[i + 1]], weight='length')

                    dis = distance(node_id_to_lat_lng[itinerary_node[i]], node_id_to_lat_lng[itinerary_node[i + 1]])
                    itinerary_segment_dis.append(dis)
                dis_array.append(sum(itinerary_segment_dis))
                itinerary_segment_dis_list.append(itinerary_segment_dis)
            itinerary_node.pop()

        # a toy example
        # for i in range(origin_coord_array.shape[0]):
        #     origin_lng = origin_coord_array[i, 0]
        #     if origin_lng == 0:
        #         itinerary_node = [0,1,2]
        #         itinerary_segment_dis = [1,1,0]
        #         dis = 2
        #     elif origin_lng == 1:
        #         itinerary_node = [2,3,0]
        #         itinerary_segment_dis = [1, 1, 0]
        #         dis = 2
        # itinerary_node_list.append(itinerary_node)
        # itinerary_segment_dis_list.append(itinerary_segment_dis)
        # dis_array.append(dis)
        # dis_array = np.array(dis_array)
    elif mode == 'drop_end':
        # 对itineray node_list和itineray segment_time_list中的各个item，把末尾节点给drop掉

        # a toy example
        # for i in range(origin_coord_array.shape[0]):
        #     origin_lng = origin_coord_array[i, 0]
        #     if origin_lng == 0:
        #         itinerary_node = [0, 1]
        #         itinerary_segment_dis = [1, 1]
        #         dis = 2
        #     elif origin_lng == 1:
        #         itinerary_node = [2, 3]
        #         itinerary_segment_dis = [1, 1]
        #         dis = 2
        # itinerary_node_list.append(itinerary_node)
        # itinerary_segment_dis_list.append(itinerary_segment_dis)
        # dis_array.append(dis)
        # dis_array = np.array(dis_array)
        for origin,dest in zip(origin_node_list, dest_node_list):
            # data = {
            #     'node': str(origin)+str(dest)
            # }
            # re = mycollect.find_one(data)['itinerary_node_list']
            # if re:
            #     ite = re
            # else:
            ite = ox.distance.shortest_path(G, origin, dest, weight='length', cpus=16)
            if ite is not None and len(ite) > 1:
                itinerary_node_list.append(ite)
            else:
                itinerary_node_list.append([origin,dest])

        for itinerary_node in itinerary_node_list:
            itinerary_segment_dis = []
            for i in range(len(itinerary_node) - 1):
                # dis = nx.shortest_path_length(G, node_id_to_lat_lng[itinerary_node[i]], node_id_to_lat_lng[itinerary_node[i + 1]], weight='length')
                try:
                    dis = distance(node_id_to_lat_lng[itinerary_node[i]], node_id_to_lat_lng[itinerary_node[i + 1]])
                except:
                    print("itinerary exception")

                itinerary_segment_dis.append(dis)
            itinerary_node.pop()
            dis_array.append(sum(itinerary_segment_dis))
            # if len(itinerary_node) != itinerary_segment_dis:
            #     print(len(itinerary_node))
            #     print(itinerary_segment_dis)
            itinerary_segment_dis_list.append(itinerary_segment_dis)

    dis_array = np.array(dis_array)
    return itinerary_node_list, itinerary_segment_dis_list, dis_array

class road_network:

    def __init__(self, **kwargs):
        self.params = kwargs

    def load_data(self):
        """
        :param data_path: the path of road_network file
        :type data_path:  string
        :param file_name: the filename of road_network file
        :type file_name:  string
        :return: None
        :rtype:  None
        """
        # 路网格式：节点数字编号（从0开始），节点经度，节点纬度，所在grid id
        self.df_road_network = result

    def get_information_for_nodes(self, node_id_array):
        """
        :param node_id_array: the array of node id
        :type node_id_array:  numpy.array
        :return:  (lng_array,lat_array,grid_id_array), lng_array is the array of longitude;
                lat_array is the array of latitude; the array of node id.
        :rtype: tuple
        """
        index_list = [self.df_road_network[self.df_road_network['node_id'] == item].index[0] for item in node_id_array]
        lng_array = self.df_road_network.loc[index_list,'lng'].values
        lat_array = self.df_road_network.loc[index_list,'lat'].values
        grid_id_array = self.df_road_network.loc[index_list,'grid_id'].values
        return lng_array, lat_array, grid_id_array

def get_exponential_epsilons(initial_epsilon, final_epsilon, steps, decay=0.99, pre_steps=10):
    """
    :param initial_epsilon: initial epsilon
    :type initial_epsilon: float
    :param final_epsilon: final epsilon
    :type final_epsilon: float
    :param steps: the number of iteration
    :type steps: int
    :param decay: decay rate
    :type decay:  float
    :param pre_steps: the number of iteration of pre randomness
    :type pre_steps: int
    :return: the array of epsilon
    :rtype: numpy.array
    """

    epsilons = []

   # pre randomness
    for i in range(0, pre_steps):
        epsilons.append(deepcopy(initial_epsilon))

    # decay randomness
    epsilon = initial_epsilon
    for i in range(pre_steps, steps):
        epsilon = max(final_epsilon, epsilon * decay)
        epsilons.append(deepcopy(epsilon))

    return np.array(epsilons)

def sample_all_drivers(driver_info, t_initial, t_end, driver_sample_ratio=1, driver_number_dist=''):
    """
    :param driver_info: the information of driver
    :type driver_info:  pandas.DataFrame
    :param t_initial:   time of initial state
    :type t_initial:    int
    :param t_end:       time of terminal state
    :type t_end:        int
    :param driver_sample_ratio:
    :type driver_sample_ratio:
    :param driver_number_dist:
    :type driver_number_dist:
    :return:
    :rtype:
    """
    # 当前并无随机抽样司机；后期若需要，可设置抽样模块生成sampled_driver_info
    new_driver_info = deepcopy(driver_info)
    sampled_driver_info = new_driver_info.sample(frac=driver_sample_ratio)
    sampled_driver_info['status'] = 3
    loc_con = sampled_driver_info['start_time'] <= t_initial
    sampled_driver_info.loc[loc_con, 'status'] = 0
    sampled_driver_info['target_loc_lng'] = sampled_driver_info['lng']
    sampled_driver_info['target_loc_lat'] = sampled_driver_info['lat']
    sampled_driver_info['target_grid_id'] = sampled_driver_info['grid_id']
    sampled_driver_info['remaining_time'] = 0
    sampled_driver_info['matched_order_id'] = 'None'
    sampled_driver_info['total_idle_time'] = 0
    sampled_driver_info['time_to_last_cruising'] = 0
    sampled_driver_info['current_road_node_index'] = 0
    sampled_driver_info['remaining_time_for_current_node'] = 0
    sampled_driver_info['itinerary_node_list'] = [[] for i in range(sampled_driver_info.shape[0])]
    sampled_driver_info['itinerary_segment_time_list'] = [[] for i in range(sampled_driver_info.shape[0])]

    return sampled_driver_info

def sample_request_num(t_mean, std, delta_t):
    """
    sample request num during delta t
    :param t_mean:
    :param std:
    :param delta_t:
    :return:
    """
    random_num = np.random.normal(t_mean, std, 1)[0] * (delta_t / 100)
    random_int = random_num // 1
    random_reminder = random_num % 1

    rn = random.random()
    if rn < random_reminder:
        request_num = random_int + 1
    else:
        request_num = random_int
    return int(request_num)

def reposition(eligible_driver_table, mode):
    """
    :param eligible_driver_table:
    :type eligible_driver_table:
    :param mode:
    :type mode:
    :return:
    :rtype:
    """
    random_number = np.random.randint(0, side * side - 1)
    dest_array = []
    for _ in range(len(eligible_driver_table)):
        record = result[result['grid_id'] == random_number]
        if len(record) > 0:
            dest_array.append([record.iloc[0]['lng'], record.iloc[0]['lat']])
        else:
            dest_array.append([result.iloc[0]['lng'], result.iloc[0]['lat']])
    coord_array = eligible_driver_table.loc[:, ['lng', 'lat']].values
    itinerary_node_list, itinerary_segment_dis_list, dis_array = route_generation_array(coord_array, np.array(dest_array))
    return itinerary_node_list, itinerary_segment_dis_list, dis_array

def cruising(eligible_driver_table, mode,grid_value ={}):
    """
    :param eligible_driver_table: information of eligible driver.
    :type eligible_driver_table: pandas.DataFrame
    :param mode: the type of both-rg-cruising, if type is random; it can cruise to every node with equal
                probability; if the type is nearby, it will cruise to the node in adjacent grid or
                just stay at the original region.
    :type mode: string
    :return: itinerary_node_list, itinerary_segment_dis_list, dis_array
    :rtype: tuple
    """
    dest_array = []
    grid_id_list = eligible_driver_table.loc[:, 'grid_id'].values
    for grid_id in (grid_id_list):
        if mode == "global-random":
            random_number = random.randint(0,side*side-1)
        elif mode == 'random':
            target = [grid_id]
            if int((grid_id - 1) / side) == int(grid_id / side) and grid_id - 1 > 0:
                target.append(grid_id - 1)
            if int((grid_id + 1) / side) == int(grid_id / side) and grid_id + 1 < side * side:
                target.append(grid_id + 1)
            if grid_id + side < side * side:
                target.append(grid_id + side)
            if grid_id - side > 0:
                target.append(grid_id - side)
            random_number = choice(target)
        elif mode == 'nearby':
            target = []
            if int((grid_id - 1) / side) == int(grid_id / side) and grid_id - 1 > 0:
                target.append(grid_id - 1)
            if int((grid_id + 1) / side) == int(grid_id / side) and grid_id + 1 < side * side:
                target.append(grid_id + 1)
            if grid_id + side < side * side:
                target.append(grid_id + side)
            if grid_id - side > 0:
                target.append(grid_id - side)
            random_number = choice(target)
        record = result[result['grid_id'] == random_number]
        if len(record) > 0:
            dest_array.append([record.iloc[0]['lng'], record.iloc[0]['lat']])
        else:
            dest_array.append([result.iloc[0]['lng'], result.iloc[0]['lat']])
    coord_array = eligible_driver_table.loc[:, ['lng', 'lat']].values
    itinerary_node_list, itinerary_segment_dis_list, dis_array = route_generation_array(coord_array,
                                                                                np.array(dest_array))
    return itinerary_node_list, itinerary_segment_dis_list, dis_array

def skewed_normal_distribution(u,thegma,k,omega,a,input_size):

    return skewnorm.rvs(a,loc=u,scale=thegma,size=input_size)

def order_dispatch(wait_requests, driver_table, maximal_pickup_distance=950, dispatch_method='LD',method='pickup_distance'):
    """
    :param wait_requests: the requests of orders
    :type wait_requests: pandas.DataFrame
    :param driver_table: the information of online drivers
    :type driver_table:  pandas.DataFrame
    :param maximal_pickup_distance: maximum of pickup distance
    :type maximal_pickup_distance: int
    :param dispatch_method: the method of order dispatch
    :type dispatch_method: string
    :return: matched_pair_actual_indexs: order and driver pair, matched_itinerary: the itinerary of matched driver
    :rtype: tuple
    """
    con_ready_to_dispatch = (driver_table['status'] == 0) | (driver_table['status'] == 4)
    idle_driver_table = driver_table[con_ready_to_dispatch]
    num_wait_request = wait_requests.shape[0]
    num_idle_driver = idle_driver_table.shape[0]
    matched_pair_actual_indexs = []
    matched_itinerary = []

    if num_wait_request > 0 and num_idle_driver > 0:
        if dispatch_method == 'LD':
            # generate order driver pairs and corresponding itinerary
            request_array_temp = wait_requests.loc[:, ['origin_lng', 'origin_lat', 'order_id', 'weight']]
            request_array = np.repeat(request_array_temp.values, num_idle_driver, axis=0)
            driver_loc_array_temp = idle_driver_table.loc[:, ['lng', 'lat', 'driver_id']]
            driver_loc_array = np.tile(driver_loc_array_temp.values, (num_wait_request, 1))
            dis_array = distance_array(request_array[:, :2], driver_loc_array[:, :2])
            if method == "pickup_distance":
                # weight转换为最大pickup distance - 当前pickup distance
                request_array[:,-1] = maximal_pickup_distance - dis_array + 1
            flag = np.where(dis_array <= maximal_pickup_distance)[0]
            if len(flag) > 0:
                order_driver_pair = np.vstack(
                    [request_array[flag, 2], driver_loc_array[flag, 2], request_array[flag, 3], dis_array[flag]]).T
                matched_pair_actual_indexs = LD(order_driver_pair.tolist())
                request_indexs = np.array(matched_pair_actual_indexs)[:, 0]
                driver_indexs = np.array(matched_pair_actual_indexs)[:, 1]
                request_indexs_new = []
                driver_indexs_new = []
                for index in request_indexs:
                    request_indexs_new.append(request_array_temp[request_array_temp['order_id'] == int(index)].index.tolist()[0])
                for index in driver_indexs:
                    driver_indexs_new.append(driver_loc_array_temp[driver_loc_array_temp['driver_id'] == index].index.tolist()[0])
                request_array_new = np.array(request_array_temp.loc[request_indexs_new])[:,:2]
                driver_loc_array_new = np.array(driver_loc_array_temp.loc[driver_indexs_new])[:,:2]
                itinerary_node_list, itinerary_segment_dis_list, dis_array = route_generation_array(
                    driver_loc_array_new, request_array_new, mode=env_params['pickup_mode'])
                # itinerary_node_list_new = []
                # itinerary_segment_dis_list_new = []
                # dis_array_new = []
                # for item in matched_pair_actual_indexs:
                #     index = int(item[3])
                #     itinerary_node_list_new.append(itinerary_node_list[index])
                #     itinerary_segment_dis_list_new.append(itinerary_segment_dis_list[index])
                #     dis_array_new.append(dis_array[index])

                matched_itinerary = [itinerary_node_list, itinerary_segment_dis_list, dis_array]
    return matched_pair_actual_indexs, np.array(matched_itinerary)

def driver_online_offline_decision(driver_table, current_time):

    # 注意pickup和delivery driver不应当下线
    # 车辆状态：0 cruise (park 或正在cruise)， 1 表示delivery，2 pickup, 3 表示下线, 4 reposition
    # This function is aimed to switch the driver states between 0 and 3, based on the 'start_time' and 'end_time' of drivers
    # Notice that we should not change the state of delievery and pickup drivers, since they are occopied.
    online_driver_table = driver_table.loc[(driver_table['start_time'] <= current_time) & (driver_table['end_time'] > current_time)]
    offline_driver_table = driver_table.loc[(driver_table['start_time'] > current_time) | (driver_table['end_time'] <= current_time)]

    online_driver_table = online_driver_table.loc[(online_driver_table['status'] != 1) & (online_driver_table['status'] != 2)]
    offline_driver_table = offline_driver_table.loc[(offline_driver_table['status'] != 1) & (offline_driver_table['status'] != 2)]
    # print(f'online count: {len(online_driver_table)}, offline count: {len(offline_driver_table)}, total count: {len(driver_table)}')
    new_driver_table = driver_table
    new_driver_table.loc[new_driver_table.isin(online_driver_table.to_dict('list')).all(axis=1), 'status'] = 0
    new_driver_table.loc[new_driver_table.isin(offline_driver_table.to_dict('list')).all(axis=1), 'status'] = 3
    # return new_driver_table
    return new_driver_table

# define the function to get zone_id of segment node

def get_nodeId_from_coordinate(lng, lat):
    """

    :param lat: latitude
    :type lat:  float
    :param lng: longitute
    :type lng:  float
    :return:  id of node
    :rtype: string
    """
    node_list = []
    for i in range(len(lat)):
        x = node_coord_to_id[(lng[i],lat[i])]
        node_list.append(x)
    return node_list

def KM_simulation(wait_requests, driver_table, method = 'nothing'):
    # currently, we use the dispatch alg of peibo
    idle_driver_table = driver_table[driver_table['status'] == 0]
    num_wait_request = wait_requests.shape[0]
    num_idle_driver = idle_driver_table.shape[0]

    if num_wait_request > 0 and num_idle_driver > 0:
        starttime_1 = time.time()

        request_array = wait_requests.loc[:, ['origin_lat', 'origin_lng', 'order_id', 'weight']].values
        request_array = np.repeat(request_array, num_idle_driver, axis=0)
        driver_loc_array = idle_driver_table.loc[:, ['lat', 'lng', 'driver_id']].values
        driver_loc_array = np.tile(driver_loc_array, (num_wait_request, 1))
        assert driver_loc_array.shape[0] == request_array.shape[0]
        dis_array = distance_array(request_array[:, :2], driver_loc_array[:, :2])
        # print('negative: ', np.where(dis_array)<0)
        flag = np.where(dis_array <= 950)[0]
        if method == 'pickup_distance':
            order_driver_pair = np.vstack([request_array[flag, 2], driver_loc_array[flag, 2], 951 - dis_array[flag], dis_array[flag]]).T
        elif method in ['total_travel_time_no_subway', 'total_travel_time_with_subway']:
            order_driver_pair = np.vstack(
                [request_array[flag, 2], driver_loc_array[flag, 2], request_array[flag, 3] + 135 - dis_array[flag]/6.33, dis_array[flag]]).T
        elif method in ['sarsa_total_travel_time', 'sarsa_total_travel_time_no_subway']:
            order_driver_pair = np.vstack(
                [request_array[flag, 2], driver_loc_array[flag, 2], request_array[flag, 3] + 135 - dis_array[flag] / 6.33,
                 dis_array[flag]]).T
        else:
            order_driver_pair = np.vstack([request_array[flag, 2], driver_loc_array[flag, 2], request_array[flag, 3], dis_array[flag]]).T  # rl for matching
        order_driver_pair = order_driver_pair.tolist()

        endtime_1 = time.time()
        dtime_1 = endtime_1 - starttime_1
        #print('# of pairs: ', len(order_driver_pair))
        #print("pair forming time：%.8s s" % dtime_1)

        if len(order_driver_pair) > 0:
            #matched_pair_actual_indexs = km.run_kuhn_munkres(order_driver_pair)
            matched_pair_actual_indexs = dispatch_alg_array(order_driver_pair)

            endtime_2 = time.time()
            dtime_2 = endtime_2 - endtime_1
            #print('# of matched pairs: ', len(matched_pair_actual_indexs))
            #print("dispatch alg 1 running time：%.8s s" % dtime_2)
        else:
            matched_pair_actual_indexs = []
    else:
        matched_pair_actual_indexs = []

    return matched_pair_actual_indexs

def KM_for_agent():
    # KM used in agent.py for KDD competition
    pass

def random_actions(possible_directions):
    # make random move and generate a one hot vector
    action = random.sample(possible_directions, 1)[0]
    return action

# rl for matching
# state for sarsa
class State:
    def __init__(self, time_slice: int, grid_id: int):
        self.time_slice = time_slice  # time slice
        self.grid_id = grid_id  # the grid where a taxi stays in

    def __hash__(self):
        return hash(str(self.grid_id) + str(self.time_slice))

    def __eq__(self, other):
        if self.grid_id == other.grid_id and self.time_slice == other.time_slice:
            return True
        return False
# rl for matching

#############################################################################


import torch
import numpy as np
from models import NN, NNPretrained

# train a one-hidden-layer neural network on given dataset

def MSE(Y, Y_pred):
    N = Y.shape[1]  # num of data
    return ((Y - Y_pred) ** 2).sum() / N

def train(X: torch.Tensor, Y: torch.Tensor, num_epoch, model: NN, early_stop_loss=0.5):
    for epoch in range(num_epoch):
        Y_pred = model.forward(X)
        loss = MSE(Y, Y_pred)
        if epoch % 50 == 0:
            print('loss at epoch {} is {}'.format(epoch, loss.item()))
        if loss < early_stop_loss:
            break
        loss.backward()
        model.optimize(momentum=0.9)

    Y_pred = model.forward(X)
    loss = MSE(Y, Y_pred)
    print('final loss at epoch {} is {}'.format(epoch, loss.item()))

def train_by_solving(X: torch.Tensor, Y: torch.Tensor, model: NNPretrained):
    # directly solve the convex program, instead of doing gradient descent
    with torch.no_grad():
        source_feature = model.sigmoid(model.W1.mm(X) + model.b1)
        SS = 0.
        YS = 0.
        s = source_feature.mean(dim=1).reshape([-1, 1])
        y = Y.mean(dim=1).reshape([-1, 1])
        N = X.shape[1]
        for i in range(N):
            SS += source_feature[:, i].reshape([-1, 1]).mm(source_feature[:, i].reshape([1, -1]))
            YS += Y[:, i].reshape([-1, 1]).mm(source_feature[:, i].reshape([1, -1]))
        SS /= N
        YS /= N
        P1 = YS - y.reshape([-1, 1]).mm(s.reshape([1, -1]))
        P2 = SS - s.reshape([-1, 1]).mm(s.reshape([1, -1]))
        model.W2 = P1.mm(P2.pinverse())
        model.b2 = y - (model.W2.mm(s)).reshape([-1, 1])

        Y_pred = model.forward(X)
        loss = MSE(Y, Y_pred)
        print('final loss is {}'.format(loss.item()))

if __name__ == "__main__":
    data_file_handle = "rbf-n-50-d-10-m-100-N-5000-num_centers-10"
    data_file = 'datasets/' + data_file_handle + '.npz'
    data = np.load(data_file)
    X = torch.Tensor(data['X'])
    Y = torch.Tensor(data['Y'])

    n = X.shape[0]  # input dim
    d = Y.shape[0]  # output dim
    # m = 60  # hidden layer dim
    lr = 0.2  # step size
    num_epoch = 10000  # max number of epoch

    m_begin = 50
    m_step = 10
    num_steps = 0
    m_stop = m_begin + num_steps * m_step + 1
    for m in range(m_begin, m_stop, m_step):
        model = NN(n=n, d=d, m=m, lr=lr)
        train(X, Y, num_epoch, model, early_stop_loss=0.)
        model_file_handle = 'rbf-n-' + str(n) + '-d-' + str(d) + '-m-' + str(m) + '-N-' + str(X.shape[1])
        model.save_model(model_file_handle)


"""
This module adds the layer 3 functionalities. It allows the creation of\
IP interface objects, with IP addresses and routes.
"""
from pyroute2.netlink.exceptions import NetlinkError
from socket import AF_INET, AF_INET6
from pyfconfig.Interface import Interface
from pyroute2 import IPRoute
from pyfconfig.exceptions import *
from pyfconfig.methods import *
from pyfconfig.constants import IS_ROOT
import ipaddress

class IpInterface(Interface):
    """
    The base class for interacting with IP enabled interfaces.
    """

    def __init__(self, name):
        """
        Creates an IPInterface object.

        :param name: The name of the interface.
               This name must reflect an existing interface such as "eth0"\
 on most systems.
        """
        super().__init__(name)

    def __repr__(self):
        return f"pyfconfig.IpInterface({self._name})"

    @property
    def ipAddresses(self):
        """
        Returns the list of IP addresses assigned to this interface.
        The list contains addresses as defined by the standard library\
ipaddress. The addresses are represented as IPv4Interface\
 or IPv6Interface objects.
        """
        returnList = []
        addresses = ()
        with IPRoute() as ipr:
            addresses = ipr.get_addr(label=self._name)
            for address in addresses:
                theip = [x[1]
                         for x in address["attrs"] if x[0] == "IFA_ADDRESS"][0]
                theprefix = address["prefixlen"]
                if address["family"] == AF_INET:
                    returnList.append(
                        ipaddress.IPv4Interface(f"{theip}/{theprefix}"))
                elif address["family"] == AF_INET6:
                    returnList.append(
                        ipaddress.IPv6Interface(f"{theip}/{theprefix}"))
        return returnList

    def addIpAddress(self, address):
        """
        Adds an IP address to the current Interface.

        :param address: The string representing the IP address.
                  Example : "10.10.10.1/24", "10.10.10.1"
        """
        if IS_ROOT:
            addr = ipaddress.ip_interface(address)
            with IPRoute() as ipr:
                index = ipr.link_lookup(ifname=self.name)[0]
                try:
                    ipr.addr("add", index=index,
                             address=str(addr.ip), mask=addr.network.prefixlen)
                except NetlinkError as e:
                    if e.code == 17:
                        raise AddressAlreadyExistsException()
                    else:
                        raise e
        else:
            raise NotRootException()

    def delIpAddress(self, address):
        """
        Removes an IP address from the current Interface.

        :param address: The string representing the IP address to remove.
                        Example : "10.10.10.1/24", "10.10.10.1".
        """
        if IS_ROOT:
            addr = ipaddress.ip_interface(address)
            with IPRoute() as ipr:
                index = ipr.link_lookup(ifname=self.name)
                try:
                    ipr.addr("delete", index=index,
                             address=str(addr.ip), mask=addr.network.prefixlen)
                except NetlinkError as e:
                    if e.code == 99:  # address already exists
                        raise AddressDoesntExistException()
                    else:
                        raise e
        else:
            raise NotRootException()

    def flushIpAddresses(self):
        """
        Flushes IP addresses for the current interface.
        """
        if IS_ROOT:
            with IPRoute() as ipr:
                ipr.flush_addr(label=self.name)
        else:
            raise NotRootException()

    @property
    def routes(self):
        """
        Returns the list of the routes having this interface as output.
        """
        routes = getRoutes()
        returnList = []
        for route in routes:
            if "outInterface" in route.keys()\
               and route["outInterface"].name == self.name:
                returnList.append(route)
        return returnList

def getRoutes(cacheInfo=False):
    """
    Returns a list of every route declared on the current system.

    :param cacheInfo: Boolean to indicate if statistical data about routes\
 must be returned if available.
    """
    returnList = []
    with IPRoute() as ipr:
        routes = ipr.get_routes()
        parsedRoutes = [parseRTAMessage(r) for r in routes]
        for route in parsedRoutes:
            if "dest" not in route.keys():
                route["dest"] = "0.0.0.0/0"
            route["dest"] = ipaddress.ip_network("0.0.0.0/0")
            if "src" in route.keys():
                route["src"] = ipaddress.ip_address(route["src"])
            if "gateway" in route.keys():
                route["gateway"] = ipaddress.ip_address(route["gateway"])
            if "outInterface" in route.keys():
                intf = ipr.get_links(route["outInterface"])[0]
                intf = intf.get_attr("IFLA_IFNAME")
                route["outInterface"] = IpInterface(intf)
            if "inInterface" in route.keys():
                intf = ipr.get_links(route["inInterface"])[0]
                intf = intf.get_attr("IFLA_IFNAME")
                route["inInterface"] = IpInterface(intf)
            if "cacheInfo" in route.keys() and cacheInfo is False:
                del route["cacheInfo"]
            returnList.append(route)
    return returnList


class Solution:

    '''
    So in this question we need to put cuts(brackets) to make a string with it's subsets as palindormes
    in the range check if it is a palindrome
    The for loop is to check all the subsets
    The condition(variable) is decided by the temp and base condition
    Everything else is the same

    A few opitmization ideas:
    1. use dp
    2. use a ispalindrome() dp to not calculate again and again if the string is palindrome or not. It will store if the current string is true or not (a palindrome)
    3. there might be a chance that the left recursive call was calculated even before the parent call was made. so we'll have to wait for the parent call to happen. What we can do is, instead calling all recursive calls together, call them one by one and check if they were already called before. Basically use (dp[i][j] != -1: retun dp[i][j]) condition for all calls
    4. They are saying if the left recursive call is not a palindrome then it won't be of any use to call the right call. As they won't contribute to the answer.
    This happens so because we are looking for an optimal answer. If left is not a palindrome it will only go towards the worst case, whereas we are tryig to go to the
    opitmal answer. So we leave this situation and move further. Hence instead of doing all recursive calls we only do those which help us in getting closer to the
    optimal answer
    '''

    def minCut(self, string: str) -> int:
        if not string or len(string) <= 1:
            return 0
        if self.palindrome(string):
            return 0

        n = len(string)

        #Optimization 1
        dp = [[-1 for i in range(n)] for j in range(n)]

        #Optimization 2
        p = [[False for i in range(n)] for j in range(n)]

        return self.solve(string, 0, n-1, dp, p)

    def solve(self, s, i , j, dp, p):
        if i>= j:
            return 0
        elif dp[i][j] != -1:
            return dp[i][j]
        elif p[i][j]:
            return 0
        elif self.palindrome(s[i:j+1]):
            p[i][j] = True
            return 0

        ans = j
        for k in range(i,j):
            #Optimization 4
            if self.palindrome(s[i:k+1]):
                temp = self.solve(s,k+1,j,dp,p) + 1
                ans = min(ans,temp)

        dp[i][j] = ans
        return dp[i][j]

    def palindrome(self, s):
        return s == s[::-1]


from django.urls import path, include

# from rest_framework.urlpatterns import format_suffix_patterns
from . import views

urlpatterns = [
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    path('register/', views.RegisterView.as_view()),
    path('login/', views.LoginView.as_view()),
    path('logout/', views.LogoutView.as_view()),

    path('users/', views.UserList.as_view()),
    path('user/<int:pk>/', views.UserDetail.as_view()),
    path('user/auth/', views.UserAuth.as_view()),

    path('lists/', views.SnippetList.as_view(), name='snippets'),
    path('detail/<int:pk>/', views.SnippetDetail.as_view()),
]


import numpy as np

def feat_engine(train_df_list):

    n_steps_in = 360
    X_cols = ['Global CMP22 (vent/cor) [W/m^2]', 'Direct sNIP [W/m^2]',
       'Azimuth Angle [degrees]', 'Tower Dry Bulb Temp [deg C]',
       'Tower Wet Bulb Temp [deg C]', 'Tower Dew Point Temp [deg C]',
       'Tower RH [%]', 'Peak Wind Speed @ 6ft [m/s]',
       'Avg Wind Direction @ 6ft [deg from N]', 'Station Pressure [mBar]',
       'Precipitation (Accumulated) [mm]', 'Snow Depth [cm]', 'Moisture',
       'Albedo (CMP11)','Total Cloud Cover [%]']

    Y_cols = ['Total Cloud Cover [%]']
    X, y = list(), list()

    y_ind_list = [n_steps_in+30, n_steps_in+60, n_steps_in+90, n_steps_in+120]
    for df in train_df_list:
        df = df.reset_index(drop=True)
        # gather input and output parts of the pattern
        seq_x, seq_y = df.loc[0:n_steps_in, X_cols], df.loc[y_ind_list, Y_cols]
        X.append(seq_x.values)
        y.append(np.concatenate(seq_y.values))
    return np.array(X), np.array(y)


from bs4 import BeautifulSoup
import requests
import csv

url_list= []
activity_ids= []
activity_names= []

start = 1
end = 33

csv_file = open("all_urls_klook.csv", 'w')
csv_writer = csv.writer(csv_file)
csv_writer.writerow(['URL', 'Title', 'Activity_ID'])

while start <= end:
    url=f'https://www.klook.com/search?query=THAILAND&price_from=0&start={start}&type=country'
    headers = {'user-agent': 'my-app/0.0.1'}
    source= requests.get(url, headers=headers).text

    soup= BeautifulSoup(source, 'lxml')

    for content in soup.find_all('a', class_='j_activity_item_link'):
        try:
            url = content['href']

            f_url = url.split('/')[2]
            full_url = f'https://www.klook.com/activity/{f_url}'
            url_list.append(full_url)

            id_from_url = f_url.split("-")[0]
            activity_ids.append(id_from_url)

            #csv_writer.writerow([full_url, "", ""])
            #print(full_url)
        except Exception as e:
            pass

    start += 1

for urls in url_list:
    url=urls
    headers = {'user-agent': 'my-app/0.0.1'}
    source= requests.get(url, headers=headers).text

    review_parse= BeautifulSoup(source, 'lxml')

    f_url = url.split('/')[4]
    activity_id = f_url.split('-')[0]
    full_url = f'https://www.klook.com/activity/{activity_id}'
    activity_ids.append(activity_id)

    print(activity_id)

    #print(full_url)

    package_name = review_parse.find('h1', class_='t32').text
    activity_names.append(package_name)

    csv_writer.writerow([full_url, package_name, activity_id])
    print(package_name)

print(url_list)


# modified version that only works with 1x1 matrices

import orbital
import math
#import scipy.linalg
#import Numeric

class fermion:
  def __init__(self,orb):
    self.orb = orb
    self.m_up = None
    self.m_down = None
    self.m_inv_up = None
    self.m_inv_down = None
  def set_orbital(self,orb):
    self.orb = orb

  def fill_matrix(self,epos):
    n = len(epos)
    e_up = epos[:(n+1)//2]
    e_down = epos[(n+1)//2:]
    self.m_up = []
    for p in e_up:
      vals = self.orb.compute_value(p,len(e_up))
      self.m_up.append(vals)

    self.m_down = []
    for p in e_down:
      vals = self.orb.compute_value(p,len(e_down))
      self.m_down.append(vals)

  def invert(self):

    if len(self.m_up) > 0:
      #self.m_inv_up = scipy.linalg.inv(self.m_up)
      self.m_inv_up = [[1.0/self.m_up[0][0]]]
    else:
      self.m_inv_up = []
    if len(self.m_down) > 0:
      #self.m_inv_down = scipy.linalg.inv(self.m_down)
      self.m_inv_down = [[1.0/self.m_down[0][0]]]
    else:
      self.m_inv_down = []

  def compute_log_ratio(self,epos,p,idx):
    log_det_old = self.compute_log_value(epos)
    old_p = epos[idx]
    epos[idx] = p
    log_det_new =  self.compute_log_value(epos)
    epos[idx] = old_p
    diff = log_det_new - log_det_old
    return diff

  def compute_log_value(self,epos):
    self.fill_matrix(epos)
    det_up = 1.0
    if len(self.m_up) == 1:
      det_up = self.m_up[0][0]
    elif len(self.m_up) > 1:
      #det_up = scipy.linalg.det(self.m_up)
      det_up = self.m_up[0][0]

    det_down = 1.0
    if len(self.m_down) == 1:
      det_down = self.m_down[0][0]
    elif len(self.m_down) > 1:
      #det_down = scipy.linalg.det(self.m_down)
      det_down = self.m_down[0][0]

    log_det =  0.0
    try:
      log_det +=  math.log(abs(det_up))
    except OverflowError:
      pass
    try:
      log_det +=  math.log(abs(det_down))
    except OverflowError:
      pass

      #log_det =  math.log(abs(det_up)) + math.log(abs(det_down))

    return log_det

  def compute_value(self,epos):
    self.fill_matrix(epos)
    det_up = 1.0
    if len(self.m_up) > 0:
      #det_up = scipy.linalg.det(self.m_up)
      det_up = self.m_up[0][0]
    det_down = 1.0
    if len(self.m_down) > 0:
      #det_down = scipy.linalg.det(self.m_down)
      det_down = self.m_down[0][0]
    return det_up*det_down

  def compute_del(self,epos,k):
    self.fill_matrix(epos)
    self.invert()
    n = len(epos)
    n_orb = len(self.m_up)
    if k > n//2:
      n_orb = len(self.m_down)

    p = epos[k]
    vals = self.orb.compute_del(p,n_orb)
    sum = []
    for i in range(len(vals[0])):
      sum.append(0.0)
    if k < (n+1)//2:
      for i in range(len(vals)):
        for j in range(len(vals[i])):
          sum[j] += vals[i][j]*self.m_inv_up[i][k]
    else:
      for i in range(len(vals)):
        for j in range(len(vals[i])):
          sum[j] += vals[i][j]*self.m_inv_down[i][k-(n+1)//2]
    return sum

  def compute_del_sq(self,epos,k):
    self.fill_matrix(epos)
    self.invert()
    n = len(epos)
    p = epos[k]
    n_orb = len(self.m_up)
    if k > n//2:
      n_orb = len(self.m_down)
    vals = self.orb.compute_del_sq(p,n_orb)
    sum = 0.0
    if k < (n+1)//2:
      for i in range(len(vals)):
        sum += vals[i]*self.m_inv_up[i][k]
    else:
      for i in range(len(vals)):
        sum += vals[i]*self.m_inv_down[i][k-(n+1)//2]
    return sum


import logging

from pytesmo.validation_framework.adapters import SelfMaskingAdapter

__logger = logging.getLogger(__name__)

'''
Get the variables that need to be loaded for filtering the data on them.
'''
def get_used_variables(filters, dataset, variable):

    variables = [ variable.pretty_name ]

    if not filters:
        return variables

    for fil in filters:
        if(fil.name == "FIL_ISMN_GOOD"):
            variables.append('soil moisture_flag')
            continue

        if((fil.name == "FIL_C3S_FLAG_0") or
           (fil.name == "FIL_C3S_NO_FLAG_1") or
           (fil.name == "FIL_C3S_NO_FLAG_2")):
            variables.append('flag')
            continue

        if((fil.name == "FIL_C3S_MODE_ASC") or
           (fil.name == "FIL_C3S_MODE_DESC")):
            variables.append('mode')
            continue

        if(fil.name == "FIL_GLDAS_UNFROZEN"):
            temp_variable = variable.pretty_name.replace("Moi", "TMP")
            variables.append(temp_variable)
            variables.append('SWE_inst')
            continue

        if((fil.name == "FIL_ASCAT_METOP_A") or
           (fil.name == "FIL_ASCAT_METOP_B")):
            variables.append('sat_id')
            continue

        if(fil.name == "FIL_ASCAT_UNFROZEN_UNKNOWN"):
            variables.append('ssf')
            continue

        if(fil.name == "FIL_ASCAT_NO_CONF_FLAGS"):
            variables.append('conf_flag')
            continue

        if(fil.name == "FIL_ASCAT_NO_PROC_FLAGS"):
            variables.append('proc_flag')
            continue

    return variables

def setup_filtering(reader, filters, dataset, variable):

    # figure out which variables we have to load because we want to use them
    load_vars = get_used_variables(filters, dataset, variable)

    # restrict the variables that are read from file in the reader
    if hasattr(reader.reader, 'parameters'):
        __logger.debug("Replacing existing variables to read: {}".format(reader.reader.parameters))
        reader.reader.parameters = load_vars

    if not filters:
        __logger.debug('No filters to apply for dataset {}.'.format(dataset))
        return reader

    filtered_reader = reader

    for fil in filters:
        __logger.debug("Setting up filter {} for dataset {}.".format(fil.name, dataset))

        if(fil.name == "FIL_ALL_VALID_RANGE"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '>=', variable.min_value, variable.pretty_name)
            filtered_reader = SelfMaskingAdapter(filtered_reader, '<=', variable.max_value, variable.pretty_name)
            continue

        if(fil.name == "FIL_ISMN_GOOD"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 'G', 'soil moisture_flag')
            continue

        if(fil.name == "FIL_C3S_FLAG_0"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 0, 'flag')
            continue

        if(fil.name == "FIL_C3S_NO_FLAG_1"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '!=', 1, 'flag')
            continue

        if(fil.name == "FIL_C3S_NO_FLAG_2"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '!=', 2, 'flag')
            continue

        if(fil.name == "FIL_C3S_MODE_ASC"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 1, 'mode')
            continue

        if(fil.name == "FIL_C3S_MODE_DESC"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 2, 'mode')
            continue

#         if(fil.name == "FIL_C3S_UNCERT"):
#             filtered_reader = SelfMaskingAdapter(filtered_reader, '<', 'sm_uncertainty', 'TODO_TRACY')

        if(fil.name == "FIL_GLDAS_UNFROZEN"):
            temp_variable = variable.pretty_name.replace("Moi", "TMP")
            filtered_reader = SelfMaskingAdapter(filtered_reader, '<', 0.001, 'SWE_inst')
            filtered_reader = SelfMaskingAdapter(filtered_reader, '>', 0.0, variable.pretty_name)
            filtered_reader = SelfMaskingAdapter(filtered_reader, '>', 1., temp_variable)
            continue

        if(fil.name == "FIL_ASCAT_METOP_A"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 3, 'sat_id')
            continue

        if(fil.name == "FIL_ASCAT_METOP_B"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 4, 'sat_id')
            continue

        if(fil.name == "FIL_ASCAT_UNFROZEN_UNKNOWN"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '<=', 1, 'ssf') ## TODO: really should be == 0 or == 1
            continue

        if(fil.name == "FIL_ASCAT_NO_CONF_FLAGS"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 0, 'conf_flag')
            continue

        if(fil.name == "FIL_ASCAT_NO_PROC_FLAGS"):
            filtered_reader = SelfMaskingAdapter(filtered_reader, '==', 0, 'proc_flag')
            continue

    return filtered_reader


import pandas as pd
import unittest

from src.ml_pipeline import correct_labels_spelling, load_data

class UnitTests(unittest.TestCase):

    def test_load_data(self):
        assert load_data("fake_path", []) is None

    def test_correct_labels_spelling(self):
        input_data = [["text_1", "Ktchen"], ["text_2", "Kitchen"]]
        input_df = pd.DataFrame(input_data, columns=['description', 'category'])
        correct_labels_spelling(input_df, "category", {"Ktchen": "Kitchen"})

        expected_data = [["text_1", "Kitchen"], ["text_2", "Kitchen"]]
        expected_df = pd.DataFrame(expected_data, columns=['description', 'category'])

        pd.testing.assert_frame_equal(input_df, expected_df)


""""
This encoder model consists of instances
1. CNN 2. Word Level 3. Phrase Level 4. Sentence Level(Referred to as Question level in paper)
This layer output the hidden state vector for both image and question at all levels to the decoder.
Co-attention part is also taken care here

"""
import tensorflow as tf
from vqa_cnn import *
from vqa_word_level import *
from vqa_phrase_level import *
from vqa_lstm import *

class vqa_encoder:
    def __init__(self,config):
        self.config = config
        self.cnn = vqa_cnn(config)
        self.word_level = vqa_word_level(config)
        self.phrase_level = vqa_phrase_level(config)
        self.sentence_level = vqa_lstm(config)

    # def build(self, images, questions, question_masks, embedding_matrix):
    def build(self, image_features, questions, question_masks, embedding_matrix):
        # ## Build the CNN model
        if self.config.PHASE == "test":
            images = image_features
            self.cnn.build(images)

        self.image_features = image_features
        ## Build the word level
        self.word_level.build(questions, question_masks, embedding_matrix)
        ## Build the Phrase level

        self.phrase_level.build(self.word_level.word_embed)
        # ## Build the sentence level LSTM Model
        self.sentence_level.build(self.phrase_level.phrase_level_features)
        # ## Combine the model

        self.build_encoder()

    def build_encoder(self):

        config = self.config
        # config.IMAGE_FEATURES = self.cnn.num_ctx

        ## d = 512, N = 14, T = 25, k = 25
        ## Building Word Level features

        if self.config.PHASE == "test":
            print("CNN feature size {}".format(self.cnn.conv_feats.get_shape())) ## [BATCH_SIZE,14,512]
            self.V = tf.transpose(self.cnn.conv_feats, [0, 2, 1]) ##[BATCH_SIZE,512,14] (V) [?,d,N]
            print("V_word shape : {}".format(self.V.get_shape()))
        else:
            print("CNN feature size {}".format(self.image_features.get_shape()))  ## [BATCH_SIZE,14,512]
            self.V = tf.transpose(self.image_features, [0, 2, 1])  ##[BATCH_SIZE,512,14] (V) [?,d,N]
            print("V_word shape : {}".format(self.V.get_shape()))

        print("Word Level feature size {}".format(self.word_level.word_embed.get_shape())) ## [BATCH_SIZE,25,512]

        self.Q_word = tf.transpose(self.word_level.word_embed,[0,2,1]) ## [BATCH_SIZE,512,25] (Q) [?,d,T]

        ## Call the parallel co-attention model on word , phrase and sentence level
        self.v_attend_word, self.q_attend_word = self.parallel_co_attention(self.V, self.Q_word, "word")

        print("Phrase Level feature size {}".format(self.phrase_level.phrase_level_features.get_shape()))
        self.Q_phrase = tf.transpose(self.phrase_level.phrase_level_features,[0,2,1])

        self.v_attend_phrase, self.q_attend_phrase = self.parallel_co_attention(self.V, self.Q_phrase, "phrase")

        print("Sentence Level feature size {}".format(self.sentence_level.lstm_features.get_shape()))
        self.Q_sentence = tf.transpose(self.sentence_level.lstm_features,[0,2,1])

        self.v_attend_sentence, self.q_attend_sentence = self.parallel_co_attention(self.V,self.Q_sentence,"sentence")

    def parallel_co_attention(self,V,Q,name_scope="word"):
        config = self.config

        with tf.variable_scope(name_scope, reuse=tf.AUTO_REUSE) as scope:
            W_b = tf.get_variable(
                initializer=tf.truncated_normal([config.EMBEDDING_DIMENSION, config.EMBEDDING_DIMENSION],
                                                dtype=tf.float32, stddev=1e-1),
                name='W_b', trainable=True)  ##[d,d] W_b

            ## Multiplying weight matrix with word level question features
            C_intermediate = tf.scan(lambda a, x: tf.matmul(x, W_b),
                                                            tf.transpose(Q, [0, 2, 1]))  ## [?,T,d]

            print("Affinity_size_intermediate {0}: {1}".format(name_scope, C_intermediate.get_shape()))

            ## Multiplyint intermediate affinity with word level image features
            C = tf.tanh(tf.matmul(C_intermediate, V))  ## (C) [?,T,N]

            print("Affinity_size {0} : {1}".format(name_scope, C.get_shape()))

            W_v = tf.get_variable(
                initializer=tf.truncated_normal([config.INTERMEDIATE_DIMENSION, config.EMBEDDING_DIMENSION],
                                                dtype=tf.float32, stddev=1e-1),
                name='W_v', trainable=True)  ## W_v [k,d]

            W_q = tf.get_variable(
                initializer=tf.truncated_normal([config.INTERMEDIATE_DIMENSION, config.EMBEDDING_DIMENSION],
                                                dtype=tf.float32, stddev=1e-1),
                name='W_q', trainable=True)  ## W_q [k,d]

            ## Calcluate the H-values
            ## W_v * V [k,d] *[?,d,N] = [?,k,N]
            W_v_V = tf.scan(lambda b, x: tf.matmul(W_v, x), V,
                                      initializer=tf.get_variable(
                                          shape=[config.INTERMEDIATE_DIMENSION, config.IMAGE_FEATURES],
                                          name="W_v_V"))

            ## W_q * Q [k,d] * [?,d,T] = [?,k,T]
            W_q_Q = tf.scan(lambda b, x: tf.matmul(W_q, x), Q,
                                      initializer=tf.get_variable(
                                          shape=[config.INTERMEDIATE_DIMENSION, config.MAX_QUESTION_LENGTH],
                                          name="W_q_Q"))

            H_v = tf.tanh( tf.add(W_v_V, tf.matmul(W_q_Q, C)))  ## [?,k,N]
            H_q = tf.tanh(tf.add(W_q_Q, tf.matmul(W_v_V, tf.transpose(C,[0, 2, 1]))))  ## [?,k,T]

            print("H_v shape {0}: {1}".format(name_scope, H_v.get_shape()))
            print("H_q shape {0}: {1}".format(name_scope, H_q.get_shape()))

            w_h_v = tf.get_variable(initializer=tf.truncated_normal([config.INTERMEDIATE_DIMENSION, 1],
                                                                              dtype=tf.float32, stddev=1e-1),
                                              name='w_h_v', trainable=True)

            w_h_q = tf.get_variable(initializer=tf.truncated_normal([config.INTERMEDIATE_DIMENSION, 1],
                                                                              dtype=tf.float32, stddev=1e-1),
                                              name='w_h_q', trainable=True)

            ## Attention weights
            a_v = tf.nn.softmax(tf.scan(lambda a, x: tf.matmul(tf.transpose(w_h_v), x), H_v,
                                    initializer=tf.get_variable(shape=[1, config.IMAGE_FEATURES],
                                                                name="a_v")) ) ## [?,1,N]

            a_q = tf.nn.softmax(tf.scan(lambda a, x: tf.matmul(tf.transpose(w_h_q), x), H_q,
                                    initializer=tf.get_variable(shape=[1, config.MAX_QUESTION_LENGTH],
                                                                name="a_q")))  ## [?,1,T]

            print("a_v shape {0} : {1}".format(name_scope,a_v.get_shape()))
            print("a_q shape {0}: {1}".format(name_scope,a_q.get_shape()))

            ## Aggregate average with attention vectors

            V_attend = tf.matmul(a_v, tf.transpose(V, [0, 2, 1]))
            Q_attend = tf.matmul(a_q, tf.transpose(Q, [0, 2, 1]))

            ## Reshaped because it gives us a dimension [?,1,d]
            V_attend = tf.reshape(V_attend, [config.BATCH_SIZE, config.EMBEDDING_DIMENSION])
            Q_attend = tf.reshape(Q_attend, [config.BATCH_SIZE, config.EMBEDDING_DIMENSION])

            print("V_attend shape {0}: {1}".format(name_scope, V_attend.get_shape()))
            print("Q_attend shape {0}: {1}".format(name_scope, Q_attend.get_shape()))

            return V_attend, Q_attend


#Example:
#Connect to two gopro cameras with given serial numbers and take - simultaneously - pictures
#For the script to work only the correct serial numbers are required
#The images are saved using given core file name, to which camera and image indices are automatically added

#************************ INPUT PARAMETERS **************************************
#Camera serial number - you can find it either under settings in the camera itself,
#or by selecting the camera and clicking "Properties" in the file explorer
SNcam1 = 'C3xxxxxxxxxxxx'
SNcam2 = 'C3yyyyyyyyyyyy'

#Core file name using which all the subsequent downloaded images will be saved
#camera number and image number will be added automatically, do not include them here
fname = 'cam'

#make sure that the cameras are connected and switched on!
#********************************************************************************

from goproUSB import GPcam
import glob
import concurrent.futures

def takePhoto(cam):
    cam.shutterStart()

cam1 = GPcam(SNcam1)
cam2 = GPcam(SNcam2)

cam1.USBenable()
cam1.modePhoto()
cam2.USBenable()
cam2.modePhoto()

with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
    results = pool.map(takePhoto, [cam1,cam2])

#wait for the camera to finish processing:
while cam1.camBusy() or cam2.camBusy():
    continue
while cam1.encodingActive() or cam2.encodingActive():
    continue

#get list of files from both cameras
ml1 = cam1.getMediaList()
ml2 = cam2.getMediaList()

#determine
fileidx = len(glob.glob(f'{fname}1_*.jpg')) + 1

cam1.mediaDownloadLast(f'{fname}1_{fileidx:03}')
cam2.mediaDownloadLast(f'{fname}2_{fileidx:03}')

cam1.USBdisable()
cam2.USBdisable()


import tensorflow as tf

def cer_loss(settings,  model_args=None, parameters=None):

    n = parameters["n"]

    def cer(y_true, y_pred):

        """ For BO, remove it for release """
        y_true = y_true[:, :settings["classes"]]
        ce = tf.keras.losses.categorical_crossentropy(y_true, y_pred)

        ce_shuffled = 0.0

        for i in range(n):
            y_true_shuffled = tf.random.shuffle(y_true)
            ce_shuffled += tf.keras.losses.categorical_crossentropy(y_true_shuffled, y_pred)

        ce_shuffled = ce_shuffled / n

        return ce / ce_shuffled

    return cer


'''배경 제거된 이미지와 배경 이미지 합성 파일(함수)'''
from torchvision import models
from PIL import Image
import matplotlib.pyplot as plt
import torch
import numpy as np
import cv2
import torchvision.transforms as T
import datetime as dt

dlab = models.segmentation.deeplabv3_resnet101(pretrained=1).eval()

#추출할 객체 이미지와 배경 합성 함수
def decode_segmap(image, source, bgimg, count, nc=21):

  label_colors = np.array([(0, 0, 0),  # 0=배경
              # 1=비행기, 2=자전거, 3=새, 4=배, 5=병
              (128, 0, 0), (0, 128, 0), (128, 128, 0), (0, 0, 128), (128, 0, 128),
              # 6=버스, 7=차, 8=고양이, 9=의자, 10=소
              (0, 128, 128), (128, 128, 128), (64, 0, 0), (192, 0, 0), (64, 128, 0),
              # 11=식탁, 12=개, 13=소, 14=오토바이, 15=사람
              (192, 128, 0), (64, 0, 128), (192, 0, 128), (64, 128, 128), (192, 128, 128),
              # 16=화분식물, 17=양, 18=소파, 19=기차, 20=TV
              (0, 64, 0), (128, 64, 0), (0, 192, 0), (128, 192, 0), (0, 64, 128)])

  #r,g,b의 값에 image와 동일한 0배열 반환
  r = np.zeros_like(image).astype(np.uint8)
  g = np.zeros_like(image).astype(np.uint8)
  b = np.zeros_like(image).astype(np.uint8)
  for l in range(0, nc):
    idx = image == l
    r[idx] = label_colors[l, 0]
    g[idx] = label_colors[l, 1]
    b[idx] = label_colors[l, 2]

  #추출할 객체를 인식하는 배열 합치기
  rgb = np.stack([r, g, b], axis=2)

  #추출할 객체 이미지 호출
  foreground = cv2.imread(source)

  #합성할 배경 이미지 호출
  background = cv2.imread(bgimg)

  #추출할 이미지를 rgb값으로 색깔 변환하고 사이즈 조정
  foreground = cv2.cvtColor(foreground, cv2.COLOR_BGR2RGB)
  background = cv2.cvtColor(background, cv2.COLOR_BGR2RGB)
  foreground = cv2.resize(foreground,(r.shape[1],r.shape[0]))
  background = cv2.resize(background,(r.shape[1],r.shape[0]))

  #uint8를 float형으로 변환
  foreground = foreground.astype(float)
  background = background.astype(float)

  #rgb 출력 맵의 이진마스크 생성
  th, alpha = cv2.threshold(np.array(rgb),0,255, cv2.THRESH_BINARY)

  #마스크에 약간의 blur처리를 통해 가장자리 부드럽게 처리
  alpha = cv2.GaussianBlur(alpha, (7,7),0)

  #알파 마스크 정규화
  alpha = alpha.astype(float)/255

  #알파 마스크와 추출할 이미지 덧붙이기
  foreground = cv2.multiply(alpha, foreground)

  #배경 이미지에 (1-alpha)값 덧붙이기
  background = cv2.multiply(1.0 - alpha, background)

  #처리된 결과물 추출
  outImage = cv2.add(foreground, background)

  #output_path = 'C:/Users/user/Desktop/project/DjangoProject/mysite/static/final/result{}.png'.format(str(dt.datetime.now()).replace(' ','').replace(':','_').replace('.','_'))
  output_path = 'C:/Users/user/Desktop/project/DjangoProject/mysite/static/final/result{}.png'.format(count)

  plt.imsave(output_path, outImage/255)

  return output_path

#이미지 사이즈 조정 및 전처리 함수
def segment(net, path, bgimagepath, count, show_orig=True, dev='cuda'):
  img = Image.open(path)

  if show_orig: plt.imshow(img); plt.axis('off'); plt.show()
  #깔끔한 결과를 위해 사이즈 조정
  trf = T.Compose([T.Resize(400),
                  T.ToTensor(),
                  T.Normalize(mean = [0.485, 0.456, 0.406],
                              std = [0.229, 0.224, 0.225])])
  inp = trf(img).unsqueeze(0).to(dev)
  out = net.to(dev)(inp)['out']
  om = torch.argmax(out.squeeze(), dim=0).detach().cpu().numpy()

  #decode_segmap 함수 실행(배경 합성)
  output_path = decode_segmap(om, path, bgimagepath, count)

  return output_path


def get_index():
    index = int(input("Podaj index -> "))
    return index

def main():
    l1 = [13, "hello", 3.14, True, None, [12, 4], 0]
    index = get_index()
    try:
        result = 1/l1[index]
        print(result)
    except TypeError:
        print(f"1/{index} Nie moge wykonać")
    except ZeroDivisionError:
        print(f"1/{index} Nie dziel przez zero")
    except IndexError:
        print(f"Nie ma indexu {index}")

if __name__ == "__main__":
    main()


from django.urls import path
from . import views
from .views import (PinCreateView, PinUpdateView, PinDeleteView,
                    UserBoardView, UserSavedPinsView, BoardCreateView,
                    BoardDetailView, BoardUpdateView, BoardDeleteView, CategoryCreateView)

urlpatterns = [
    path('', views.home, name='pin-home'),
    path('pins/today/', views.today_pins_list, name='today'),
    # path('pin/<int:pk>/', PinDetailView.as_view(), name='pin-detail'),
    path('pin/<int:pk>/', views.pin_detail, name='pin-detail'),
    path('pin/new/', PinCreateView.as_view(), name='pin-create'),
    path('pin/<int:pk>/update/', PinUpdateView.as_view(), name='pin-update'),
    path('pin/<int:pk>/delete', PinDeleteView.as_view(), name='pin-delete'),
    path('search/', views.SearchResultView, name='search'),
    path('user/created/<str:username>/', UserBoardView.as_view(), name='user-board'),
    path('user/save/pin/<int:pk>/', views.save_pin_view, name='save-pin'),
    path('user/saved/<str:username>/', UserSavedPinsView.as_view(), name='user-save-board'),
    path('board/new/', BoardCreateView.as_view(), name='board-create'),
    path('user/save/pin/<int:p_id>/board/<int:b_id>/', views.save_pin_board_view, name='save-to-board'),
    path('board/<int:pk>/', BoardDetailView.as_view(), name='board-detail'),
    path('board/<int:pk>/update/', BoardUpdateView.as_view(), name='board-update'),
    path('board/<int:pk>/delete', BoardDeleteView.as_view(), name='board-delete'),
    path('pin/<int:pk>/comment/', views.comment_create, name='pin-add-comment'),
    path('pin/category/new/', CategoryCreateView.as_view(), name='category-create'),
]


# -*- coding: utf-8 -*-
"""
Created on Fri Dec 18 14:46:52 2020

@author: User
"""

def listCross2(l1, l2):
    output_list = []
    for el_l1 in l1:
        for el_l2 in l2:
            if el_l1[:2] == el_l2[:2]:                      # [:2] two first letters - match
                output_list.append(el_l2)                   # appending words with a match to output list
                output_list.append(el_l1)
    output_list = list(dict.fromkeys(output_list))          # deleting duplicates from output_list
    output_list.sort()
    output_list.reverse()
    return output_list

# TESTING
# l1 = ['Anna', 'Monika', 'Marek', 'Janusz', 'Aniela']
# l2 = ['Gabriela', 'Marta', 'Maria', 'Antonina']
#
# test = listCross2(l1, l2)
# print(test)

"""
Returns sorted list of the words which begin with the same
characters, common for two lists.

Parameters
----------
l1 : list of strings
l2 : list of strings

Returns
-------
Reversely sorted, unique list of words from the lists, that have the
corresponding words beginning with the same two letters in another
list, e.g.:
    l1 = ['Anna','Monika', 'Marek', 'Janusz', 'Aniela']
    l2 = ['Gabriela', 'Marta', 'Maria','Antonina']
    -> ['Marta', 'Marek', 'Maria', 'Antonina', 'Anna', 'Aniela']
"""


from ..config import db

def get_user_public_id(uid):
    map_ref = db.collection(u'general').document(
        "user-verification").collection(uid).document("map")
    try:
        map_data = map_ref.get().to_dict()
        return map_data["public_id"]
    except:
        return None


from logging import info
from copy import deepcopy
from os import path

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

class ChangeHandler(FileSystemEventHandler):
    def __init__(self, callback):
        self.callback = callback
    def on_modified(self, event):
        if not event.is_directory and event.event_type == "modified":
            self.callback(event.src_path)

class Watcher(object):
    def _on_file_changed(self, src_path):
        src_path = path.relpath(src_path, self.root)
        # prevent onchange handler from being repeatedly fired by mirrors
        used_onchanges = []
        for out in self.outputs:
            props = self.file_set[out]
            files = props['files']
            callbacks = props['callbacks']
            if src_path in files:
                if 'onchange' in callbacks and not callbacks['onchange'].__name__ in used_onchanges:
                    if not callbacks['onchange'](path.abspath(path.join(self.root, src_path))):
                        raise Exception('onchange callback errored for file ' + src_path)
                    used_onchanges.append(callbacks['onchange'].__name__)
                self._compile(files, callbacks, path.abspath(out))

    def _compile(self, files, callbacks, out):
        if 'mode' in callbacks and callbacks['mode'] is 'slurp':
            slurpy = []
            for filename in files:
                try:
                    filename = path.abspath(path.join(self.root, filename))
                    with open(filename) as infile:
                        slurpy.append([filename, infile.read()])
                except IOError:
                    info('IOError on ' + filename + ', maybe the file doesnt exist')
                    pass
            if 'each' in callbacks:
                callbacks['each'](out, slurpy)
            else:
                raise Exception('slurp mode requires an each handler')
        else:
            # default mode is concat, also contains break for files_only mode
            if 'each' in callbacks:
                rv = callbacks['each']([out, files])
                if isinstance(rv, list):
                    files = rv
            if 'mode' in callbacks and callbacks['mode'] is 'files_only':
                info("files_only mode, no concat or post")
                return
            with open(out, 'w') as outfile:
                for filename in files:
                    filename = path.abspath(path.join(self.root, filename))
                    try:
                        with open(filename) as infile:
                            for line in infile:
                                outfile.write(line if not 'line' in callbacks else callbacks['line'](line))
                    except IOError:
                        info('IOError on ' + filename + ', maybe the file doesnt exist')
                        pass
            info("Wrote file: " + out)
            if 'post' in callbacks:
                callbacks['post'](out)

    def compile(self):
        for out in self.outputs:
            props = self.file_set[out]
            files = props['files']
            callbacks = props['callbacks']
            if not 'no_out' in callbacks or not callbacks['no_out']:
                self._compile(files, callbacks, path.abspath(out))

    def add_file_set(self, out, files, callbacks = {}):
        if out in self.file_set:
            raise Exception('out file is already in use by another FileSet')
        if out in files:
            raise Exception('cannot watch out file')
        self.outputs.append(out)
        self.file_set[out] = {'files' : files, 'callbacks' : callbacks}
        return out

    def add_mirror_set(self, orig, new, addl = None):
        if type(orig) is not str:
            raise TypeError('orig list is a required  argument')
        if type(new) is not str:
            raise TypeError('new is a required argument')
        # is deepcopy necessary for inline substitution later?
        files = deepcopy(self.file_set[orig]['files'])
        skin_path = path.join(self.root, new)
        out = path.join(skin_path, orig)
        callbacks = deepcopy(self.file_set[orig]['callbacks'])
        for i, filename in enumerate(files):
            if 'prefix_chomp' in callbacks:
                if filename.split('/')[0] == callbacks['prefix_chomp']:
                    filename = filename[(len(callbacks['prefix_chomp'])+1):]
            filePath = path.join(skin_path, filename)
            if (path.exists(filePath)):
                # is lineline substution while looping acceptable?
                files[i] = path.relpath(filePath, self.root)
        if addl is not None:
            files.extend(addl)
        callbacks['no_out'] = False
        self.outputs.append(out)
        self.file_set[out] = {'files' : files, 'callbacks' : callbacks}

    def stop(self):
        self.observer.stop()

    def start(self):
        event_handler = ChangeHandler(self._on_file_changed)
        self.observer = Observer()
        self.observer.schedule(event_handler, self.root, recursive=True)
        self.observer.start()

    def __init__(self, root):
        self.root = path.abspath(root)
        self.file_set = {}
        self.outputs = []