'''
0408 Multiple DTW -> Get their index order
'''
__author__ = 'Victoria'
# 2018-03-05

import numpy as np
import pandas as pd
import os
import re
from datetime import timedelta
from math import exp, sqrt
from itertools import combinations, permutations
import json
import matplotlib.pyplot as plt
from src.map_matching.read_data import get_wuhu_cellSheet, get_road_tests
from src.pre_processing.road_network import Graph, get_graph, Vertex
from src.pre_processing.read_data import get_cellSheet
from src.pre_processing.funs import cal_distance, get_datetime, output_geo_js, output_data_js, interpolate
from src.map_matching.evaluation import get_test_obs
from src.map_matching.CTrack import CTrack
from src.work.oneuser_alldata_0307 import trajectories_tojson , save_json, save_js
from src.single_analysis.user_trajectory import deal_same_obs

class Multipath_HMM(CTrack):
    def __init__(self,
                 obses,
                 g: Graph,
                 interpolate_interval=1,
                 smoothing=0,
                 is_direction=False,
                 direction_window=5,
                 penalty=0.9,
                 fill_num=None,
                 interpolate=True):
        '''
        Initialize CTrack
        :param obses: A list of DataFrame [time, lon, lat], time spacing may not be one second.
        :param g: Road network [Graph]
        :param interpolate_interval: Interval seconds of processed observables. Default = 1.
        :param smoothing: Window length of smoothing. If smoothing == 0, do not smooth.
        :param penalty: Direction penalty used in transition score.
        :param interpolate: Whether to interpolate trajectory.
        :var self.obs: The interpolated observables [DataFrame]
        :var self.N: The number of road segments
        :var self.T: The length of observables time. (The row number of self.obs)
        :var self.roads: DataFrame [index, road<Vertex>], index is pos<String: '$lon|$lat'>.
                         Roads has been already shrunken.
        '''
        self.obses = obses
        self.mean_obs = Observations(obses).get_mean_obs(fill_num)
        super().__init__(self.mean_obs, g, interpolate_interval, smoothing,
                         is_direction, direction_window, penalty, interpolate)

class DTW:
    '''
    Dynamic Time Wrapping Algorithm.
    '''
    def __init__(self, ts1, ts2, is_norm=False):
        '''
        :param ts1: Time series 1 with m time periods. A list [[lon, lat]].
        :param ts2: Time series 2 with n time periods. A list.
        :param is_norm: whether normalization before dtw.
        '''
        if is_norm:
            self.ts1 = self.normalization(ts1)
            self.ts2 = self.normalization(ts2)
        else:
            self.ts1 = ts1
            self.ts2 = ts2
        self.ots1, self.ots2 = ts1, ts2
        self.m = len(ts1)
        self.n = len(ts2)
        self.directions = [[0, 1], [1, 1], [1, 0]]

    def normalization(self, ts):
        ts = np.matrix(ts).T
        m = ts.mean(axis=1)
        std = ts.std(axis=1)
        ts[0] = (ts[0] - m[0]) / std[0]
        ts[1] = (ts[1] - m[1]) / std[1]
        return ts.T.tolist()

    def dtw(self):
        mat = np.ones((self.m, self.n)) + np.inf
        mat_ind = {}
        start_nodes = [(0, 0)]
        mat[start_nodes[0]] = 0
        layer_cnt = 0
        end_node = (self.m-1, self.n-1)

        print('m = %d, n = %d' % (self.m, self.n))
        while len(start_nodes) != 0:
            next_nodes = []
            for node in start_nodes:
                for dir in self.directions:
                    succ = (node[0] + dir[0], node[1] + dir[1])
                    if 0 <= succ[0] < self.m and 0 <= succ[1] < self.n:
                        new_dist = mat[node] + cal_distance(self.ts1[succ[0]], self.ts2[succ[1]])
                        if succ in mat_ind.keys():
                            if new_dist < mat[succ]:
                                mat_ind[succ] = [node]
                                mat[succ] = new_dist
                            elif new_dist == mat[succ]:
                                mat_ind[succ].append(node)
                        else:
                            mat_ind[succ] = [node]
                            mat[succ] = new_dist
                        if succ not in next_nodes:
                            next_nodes.append(succ)
            start_nodes = next_nodes
            layer_cnt += 1
            print('Layer %d is finished' % layer_cnt)

        print('Minimum distance = %f' % mat[end_node])

        return mat, mat_ind

    def is_in_bound(self, i, j):
        return 0 <= i < self.m and 0 <= j < self.n

    def dtw_boundary(self):
        '''
        Get a rough upper boundary
        :return:
        '''
        min_dist = 0
        node = [0, 0]
        # for i in range(max(self.m, self.n)-1):
        #     nxt = [node[0] + 1, node[1] + 1]
        #     if nxt[0] >= self.m:
        #         nxt[0] = self.m - 1
        #     if nxt[1] >= self.n:
        #         nxt[1] = self.n - 1
        #     max_dist += cal_distance(self.ts1[nxt[0]], self.ts2[nxt[1]])
        #     node = nxt
        while node != [self.m - 1, self.n - 1]:
            value = np.inf
            for dir in self.directions:
                tmp = [node[0] + dir[0], node[1] + dir[1]]
                if not self.is_in_bound(tmp[0], tmp[1]):
                    continue
                tmp_value = cal_distance(self.ts1[tmp[0]], self.ts2[tmp[1]])
                if tmp_value < value:
                    nxt = tmp
                    value = tmp_value
            min_dist += value
            node = nxt

        return min_dist

    def get_dist_matrix(self, ts1, ts2):
        dist_mat = np.zeros((self.m, self.n))
        for i in range(self.m):
            for j in range(self.n):
                dist_mat[i, j] = cal_distance(ts1[i], ts2[j])
        return dist_mat

    @staticmethod
    def get_derivatives(ts):
        ts.insert(0, ts[0])
        ts.append(ts[-1])
        dts = [[(ts[i+1][0] - ts[i-1][0])/2, (ts[i+1][1] - ts[i-1][1])/2] for i in range(1, len(ts) - 1)]
        return dts

    def get_derivatives_dist_matrix(self, ts1, ts2):
        dts1 = self.get_derivatives(ts1)
        dts2 = self.get_derivatives(ts2)
        return self.get_dist_matrix(dts1, dts2)

    def quick_dtw(self, is_derivatives=False):
        mat = np.ones((self.m, self.n)) + np.inf
        if is_derivatives:
            dist_mat = self.get_derivatives_dist_matrix(self.ts1, self.ts2)
        else:
            dist_mat = self.get_dist_matrix(self.ts1, self.ts2)
        mat_ind = {}
        start_nodes = [(0, 0)]
        end_node = (self.m - 1, self.n - 1)
        mat[start_nodes[0]] = 0
        layer_cnt = 0

        max_dist = self.dtw_boundary()

        print('m = %d, n = %d, max_dist = %f' % (self.m, self.n, max_dist))
        while len(start_nodes) != 0:
            next_nodes = []
            for node in start_nodes:
                if max_dist < mat[node] < np.inf:
                    continue
                for dir in self.directions:
                    succ = (node[0] + dir[0], node[1] + dir[1])
                    if 0 <= succ[0] < self.m and 0 <= succ[1] < self.n:
                        new_dist = mat[node] + dist_mat[succ]
                        if new_dist > max_dist:
                            continue
                        if succ in mat_ind.keys():
                            if new_dist < mat[succ]:
                                mat_ind[succ] = [node]
                                mat[succ] = new_dist
                            elif new_dist == mat[succ]:
                                mat_ind[succ].append(node)
                        else:
                            mat_ind[succ] = [node]
                            mat[succ] = new_dist
                        if succ not in next_nodes:
                            next_nodes.append(succ)
            start_nodes = next_nodes
            layer_cnt += 1
            if len(mat[mat != np.inf]) != 0:
                print('Layer %d is finished. Max = %f' % (layer_cnt, mat[mat != np.inf].max()))

        print('Minimum distance = %f' % mat[end_node])

        return mat, mat_ind

    def shortest_path_dtw(self, is_derivatives=False):
        '''
        Solving DTW with shortest path algorithm
        :param is_derivatives:
        :return:
        '''
        mat = np.ones((self.m, self.n)) + np.inf
        if is_derivatives:
            dist_mat = self.get_derivatives_dist_matrix(self.ts1, self.ts2)
        else:
            dist_mat = self.get_dist_matrix(self.ts1, self.ts2)
        mat_ind = {}
        mat[0, 0] = 0

        for layer in range(1, self.m + self.n):
            for i in range(layer+1):
                node = (i, layer-i)
                if not self.is_in_bound(node[0], node[1]):
                    continue
                dist = dist_mat[node]
                for dir in self.directions:
                    lst_node = (node[0] - dir[0], node[1] - dir[1])
                    if not self.is_in_bound(lst_node[0], lst_node[1]):
                        continue
                    tmp_value = mat[lst_node] + dist
                    if tmp_value < mat[node]:
                        mat[node] = tmp_value
                        mat_ind[node] = [lst_node]
                    elif tmp_value == mat[node]:
                        mat_ind[node].append(lst_node)

        # print('Minimum distance = %f' % mat[self.m-1, self.n-1])

        return mat, mat_ind

    def get_trace(self, mat_ind):
        '''
        Now only get one best trace
        :param mat_ind:
        :return:
        '''
        trace = [(self.m-1, self.n-1)]
        node = trace[0]
        while node != (0, 0):
            last_node = mat_ind[node][0]
            trace.append(last_node)
            node = last_node

        # Print trace
        trace_mat = np.zeros((self.m, self.n))
        for t in trace:
            trace_mat[t[0], t[1]] = 1
        # print(trace_mat)

        return trace, trace_mat

    def get_new_ts(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        trace, _ = self.get_trace(mat_ind)
        ts1, ts2 = self.ts1, self.ts2
        nts1, nts2 = [ts1[0]], [ts2[0]]
        for t in trace[::-1]:
            nts1.append(ts1[t[0]])
            nts2.append(ts2[t[1]])
        return nts1, nts2

    def get_average_dist(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        nts1, nts2 = self.get_new_ts(mat_ind)
        dist = 0
        for t in range(len(nts1)):
            dist += cal_distance(nts1[t], nts2[t])
        return dist / len(nts1)

    def get_median_dist(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        nts1, nts2 = self.get_new_ts(mat_ind)
        dist = []
        for t in range(len(nts1)):
            dist.append(cal_distance(nts1[t], nts2[t]))
        return np.median(dist)

    def get_middle_ts(self, mat_ind=None):
        nts1, nts2 = self.get_new_ts(mat_ind)
        return ((np.array(nts1) + np.array(nts2)) / 2).tolist()

    def plot(self, trace, title='DTW'):
        ts1 = np.matrix(self.ots1).T.tolist()
        ts2 = np.matrix(self.ots2).T.tolist()
        plt.title(title)
        plt.plot(ts1[0], ts1[1], color='blue', linewidth="0.8")
        plt.plot(ts2[0], ts2[1], color='red', linewidth="0.8")
        for line in trace:
            p1 = self.ots1[line[0]]
            p2 = self.ots2[line[1]]
            plt.plot([p1[0], p2[0]], [p1[1], p2[1]], color='black', linewidth="0.4")
        plt.grid(True, linestyle="--", color="black", linewidth="0.4")
        plt.savefig('../../res/work/0401_ddtw/pic/%s.png' % title)
        plt.show()

class MultiDTW:
    def __init__(self, tses):
        '''
        Multiple trajectories dynamic time wrapping
        :param tses: A list of ts. [ts1, ..., tsn]. tsi = [[lon, lat]]
        :var self.k: Number of time series
        :var self.dim: Length of each time series
        :var self..directions: All direction allowed
        '''
        self.tses = tses
        self.k = len(tses)
        self.dim = tuple([len(ts) for ts in tses])
        self.directions = self.get_directions(len(tses))

    @staticmethod
    def dtw_tses_single_time(tses):
        '''
        Do DTW between each two time series, and return a list of new time series.
        :param tses:
        :return:
        '''
        inds = list(combinations(np.arange(len(tses)), 2))
        ntses = []
        for ind in inds:
            ts1 = tses[ind[0]]
            ts2 = tses[ind[1]]
            dtw = DTW(ts1, ts2)
            ntses.append(dtw.get_middle_ts())

    @staticmethod
    def get_directions(k):
        '''
        Get valid directions with dimension k.
        :param k:
        :return: A list.
        e.x: k = 2
        return [[1, 0], [0, 1], [1, 1]]
        k = 3
        return [[1, 0, 0], [0, 1, 0], [1, 1, 1]
        '''
        res = list(set(combinations([0, 1]*k, k)))
        res.remove(tuple([0]*k))
        return res

    def is_in_bound(self, pos):
        for k in range(len(self.dim)):
            if not 0 <= pos[k] < self.dim[k]:
                return False
        return True

    def cal_multi_distance(self, ind):
        '''
        ind is a tuple
        :param ind:
        e.x:
            ind = (1, 2, 4)
            Calculate distance between ts0_1, ts1_2, ts2_4.
        :return:
        '''
        ind = list(ind)
        combs = list(combinations(range(len(ind)), 2))
        dist = 0
        for comb in combs:
            ts1_ind = comb[0]
            ts2_ind = comb[1]
            ts1_t = ind[ts1_ind]
            ts2_t = ind[ts2_ind]
            dist += cal_distance(self.tses[ts1_ind][ts1_t], self.tses[ts2_ind][ts2_t])
        return dist / len(combs)

    def dtw_tses(self):
        '''
        Multiple DTW
        :return:
        '''
        mat = np.ones(self.dim) + np.inf
        mat_ind = {}
        mat[tuple([0]*self.k)] = 0

        for layer in range(1, sum(self.dim)):
            inds = self.get_ksum(layer)
            for node in inds:
                if not self.is_in_bound(node):
                    continue
                dist = self.cal_multi_distance(node)
                for dir in self.directions:
                    lst_node = tuple((np.array(node) - np.array(dir)).tolist())
                    if not self.is_in_bound(lst_node):
                        continue
                    tmp_value = mat[lst_node] + dist
                    if tmp_value < mat[node]:
                        mat[node] = tmp_value
                        mat_ind[node] = [lst_node]
                    elif tmp_value == mat[node]:
                        mat_ind[node].append(lst_node)
            if len(mat[mat != np.inf]) != 0:
                print('Layer %d is finished. Max = %f' % (layer, mat[mat != np.inf].max()))

        print('Minimum distance = %f' % mat[tuple([-1]*self.k)])

        return mat, mat_ind

    def get_trace(self, mat_ind=None):
        '''
        Now only get one best trace
        :param mat_ind:
        :return:
        '''
        if mat_ind is None:
            _, mat_ind = self.dtw_tses()
        trace = [tuple([k-1 for k in self.dim])]
        node = trace[0]
        while node != tuple([0]*self.k):
            last_node = mat_ind[node][0]
            trace.append(last_node)
            node = last_node

        return trace

    def plot(self, title='DTW of multiple TS', trace=None):
        if trace is None:
            trace = self.get_trace()
        plt.title(title)
        colors = ['red', 'blue', 'cyan', 'orange']
        lin_colors = ['black', 'brown', 'purple', 'gray']
        for i, ts in enumerate(self.tses):
            ts = np.matrix(ts).T.tolist()
            plt.plot(ts[0], ts[1], color=colors[i % len(colors)], linewidth="2")

        for i, line in enumerate(trace):
            combs = list(combinations(range(len(line)), 2))
            for comb in combs:
                ts1_ind = comb[0]
                ts2_ind = comb[1]
                ts1_t = line[ts1_ind]
                ts2_t = line[ts2_ind]
                p1 = self.tses[ts1_ind][ts1_t]
                p2 = self.tses[ts2_ind][ts2_t]
                plt.plot([p1[0], p2[0]], [p1[1], p2[1]], color=lin_colors[i % len(lin_colors)], linewidth="0.5")
        plt.grid(True, linestyle="--", color="black", linewidth="0.4")
        plt.savefig('../../res/work/0401_ddtw/pic/%s.png' % title)
        plt.show()

    def get_ksum(self, n):
        res = get_ksum(n, self.k)
        return [tuple(x) for x in res]

class MultiDTWIndx:
    def __init__(self, obses, interval=1):
        '''
        Get all the obses' index order
        :param obses: Input a list of DataFrame -> [pd.DataFrame(['dates', 'lon', 'lat'])]
                      Must ordered by dates.
        :param interval: Interpolate interval. Default 1.
        '''
        self.obses = self.set_origin_destination(obses)
        self.tses, self.new_obses = self.interpolate(interval)
        self.interval = interval

    def set_origin_destination(self, obses):
        '''
        Set all the observations have the same origin point and destination point
        :param obses: [DataFrame['lon', 'lat', 'dates']]
        :return: A new processed obses
        '''
        origins = [[obs.lon[0], obs.lat[0]] for obs in obses]
        destinations = [[list(obs.lon)[-1], list(obs.lat)[-1]] for obs in obses]
        origin = [np.mean([x[0] for x in origins]), np.mean([x[1] for x in origins])]
        destination = [np.mean([x[0] for x in destinations]), np.mean([x[1] for x in destinations])]
        for i in range(len(obses)):
            obses[i].lon[0] = origin[0]
            obses[i].lat[0] = origin[1]
            obses[i].lon[len(obses[i])-1] = destination[0]
            obses[i].lat[len(obses[i]) - 1] = destination[1]
        return obses

    def interpolate(self, interval):
        '''
        Interpolate all the observations with the given interval.
        :param interval:
        :return:
        '''
        tses = []
        new_obses = []
        for obs in self.obses:
            ts, tmp = self.interpolate_single(obs, interval)
            new_obses.append(tmp)
            tses.append(ts)
        return tses, new_obses

    @staticmethod
    def interpolate_single(obs, interval):
        '''
        Interpolate one observation with the given interval.
        :param obs: DataFrame
        :param interval:
        :return: ts: [[lon, lat]]; new_obs: DataFrame
        '''
        new_obs = interpolate(obs, interval, origin_reserve=True)
        ts = [[new_obs.lon[i], new_obs.lat[i]] for i in range(len(new_obs))]
        return ts, new_obs

    def run(self):
        ts0 = self.tses[0]
        new_obs = self.new_obses[0]
        # new_obs.lon = [round(x, 7) for x in new_obs.lon]
        # new_obs.lat = [round(x, 7) for x in new_obs.lat]
        new_obs['traj'] = -1
        for i, ts in enumerate(self.tses):
            obs = self.obses[i]
            dtw = DTW(ts0, ts, is_norm=False)
            nts0, nts = dtw.get_new_ts()
            # nts = [[round(x[0], 7), round(x[1], 7)] for x in nts]
            # nts0 = [[round(x[0], 7), round(x[1], 7)] for x in nts0]
            # obs.lon = [round(x, 7) for x in obs.lon]
            # obs.lat = [round(x, 7) for x in obs.lat]
            for t in range(len(obs)):
                # if [obs.lon[t], obs.lat[t]] in nts:
                ind = nts.index([obs.lon[t], obs.lat[t]])
                ind2 = (new_obs.lon == nts0[ind][0]) & (new_obs.lat == nts0[ind][1])
                new_obs.lon[ind2] = obs.lon[t]
                new_obs.lat[ind2] = obs.lat[t]
                new_obs.traj[ind2] = i
            print(new_obs[round(new_obs.lon, 6) == 117.229595])
            print(len(new_obs))
        new_obs = new_obs[new_obs.traj != -1]
        return new_obs

    def run_one_by_one(self):
        '''
        Get the fused observations.
        Method:
            1. Take obses[0] and obses[1], interpolate them and get the ts format.
            2. Fuse them into a new observation(only have observation points, no interpolate points).
            3. Take the new observation and obses[k+1], repeat Step 1 and Step 2.
        :return: A fused observation: DataFrame['lon', 'lat', 'dates']
        '''
        new_obs = self.obses[0]

        for i in range(1, len(self.obses)):
            obs = self.obses[i]
            ts0, obs0 = self.interpolate_single(new_obs, self.interval)
            ts, _ = self.interpolate_single(obs, self.interval)
            dtw = DTW(ts0, ts, is_norm=False)
            nts0, nts = dtw.get_new_ts()

            obs0['traj'] = 0
            for t in range(len(obs)):
                ind = nts.index([obs.lon[t], obs.lat[t]])
                ind2 = (obs0.lon == nts0[ind][0]) & (obs0.lat == nts0[ind][1])
                obs0.lon[ind2] = obs.lon[t]
                obs0.lat[ind2] = obs.lat[t]
                obs0.traj[ind2] = 1
            obs0 = obs0[obs0.traj == 1]
            obs0 = obs0.drop(['traj'], axis=1)
            new_obs = pd.concat([new_obs, obs0])
            # in deal_same_obs already has sort_values() and reset_index() method
            new_obs = deal_same_obs(new_obs)

        return new_obs[['lon', 'lat', 'dates']]

    def plot(self, new_obs=None):
        if new_obs is None:
            new_obs = self.run_one_by_one()
        print(new_obs)
        plt.title('DTW Index Sort')
        for obs in self.obses:
            plt.plot(obs.lon, obs.lat, linewidth="3")

        plt.plot(new_obs.lon, new_obs.lat, color='red', linewidth="1")

        plt.show()

def get_ksum(n, k, tmp_number=[], res=[]):
    '''
    K natural numbers sums up is n. Already given res (a list of natural numbers)
    :param n: The sum of k numbers.
    :param k: Number of natural numbers.
    :param tmp_number: Now the given numbers.
    :param res: The Final result. Always input []
    :return:
    '''
    if len(tmp_number) == k - 1:
        tmp_number.append(n - sum(tmp_number))
        return [tmp_number]
    for i in range(n-sum(tmp_number)+1):
        ntmp_number = tmp_number.copy()
        ntmp_number.append(i)
        a = get_ksum(n, k, ntmp_number, res)
        if len(tmp_number) == k - 2:
            res.extend(a)
        else:
            res = a

    return res

class Observations:
    def __init__(self, obses):
        '''
        List of observations.
        :param obses: A list of DataFrame [dates, lon, lat].
        :var self.obses: A list of Observation.
        '''
        self.obses = []
        for i in range(len(obses)):
            self.obses.append(Observation(obses[i]))
        self.mean_obs = self.get_mean_obs()

    def plot_obses(self):
        for obs in self.obses:
            plt.plot(obs.lon, obs.lat)

    def len_obses(self):
        l = []
        for obs in self.obses:
            l.append(len(obs))
        return l

    def fill_with_dist(self, num):
        '''
        Let all observations filled with distanc
        :param num:
        :return:
        '''
        obses = self.obses
        res = []
        for obs in obses:
            res.append(obs.fill_with_dist(num))
        return res

    def plot_obs(self, fill_num=None):
        data = self.mean_obs
        fig = plt.figure()
        # plt.show()
        ax = fig.add_subplot(1, 1, 1)
        plt.grid(True)  # 添加网格
        plt.ion()  # interactive mode on
        for i in range(len(self.obses)):
            obs = self.obses[i].obs
            ax.plot(obs.lon, obs.lat)
            plt.pause(0.2)
        for i in range(0, len(data), 5):
            ax.plot(data.lon[:i], data.lat[:i], color='red')
            plt.pause(0.01)
        plt.pause(5)

    def get_mean_obs(self, fill_num=None):
        if fill_num is None:
            total_dist = []
            for obs in self.obses:
                total_dist.append(obs.get_total_dist())
            fill_num = np.mean(total_dist) // 10
            print('Fill num = %d' % fill_num)

        obses = self.fill_with_dist(fill_num)
        lon = []
        lat = []

        for t in range(len(obses[0])):
            lon.append(np.mean([obs.lon[t] for obs in obses]))
            lat.append(np.mean([obs.lat[t] for obs in obses]))

        return pd.DataFrame({'lon': lon, 'lat': lat, 'dates': obses[0].dates})

    def save_json(self, save_path, user_id='', is_json=True, fill_num=None, need_mean=True):
        obses = []

        for i in range(len(self.obses)):
            tmp = self.obses[i].obs
            obses.append(tmp)

        if need_mean:
            obses.append(self.get_mean_obs(fill_num))

        trajectories_tojson(obses, user_id, save_path, is_json=is_json)

class Observation:
    def __init__(self, obs):
        '''
        :param obs: A DataFrame observation [dates, lon, lat]
        '''
        self.obs = obs

    def get_obs_len(self):
        return len(self.obs)

    def get_total_dist(self):
        dist = 0
        for i in range(1, len(self.obs)):
            dist += cal_distance([self.obs.lon[i-1], self.obs.lat[i-1]],
                                 [self.obs.lon[i], self.obs.lat[i]])
        return dist

    def fill_with_dist(self, num):
        '''
        Fill the observation with distance.
        :param num: The num of final observation.
        :return: A new observation.
        '''
        total_dist = self.get_total_dist()
        each_dist = total_dist / (num - 1)

        dist = 0

        obs = self.obs
        lon = [obs.lon[0]]
        lat = [obs.lat[0]]
        dates = [obs.dates[0]]
        for i in range(1, len(obs)):
            p1 = [obs.lon[i-1], obs.lat[i-1]]
            p2 = [obs.lon[i], obs.lat[i]]
            d_have = cal_distance(p1, p2)
            d_need = each_dist - dist

            while d_have >= d_need:
                p_need = d_need / d_have
                p1 = [(1 - p_need) * p1[0] + p_need * p2[0], (1 - p_need) * p1[1] + p_need * p2[1]]
                lon.append(p1[0])
                lat.append(p1[1])
                dates.append(obs.dates[i-1] + (obs.dates[i] - obs.dates[i-1]) * p_need)

                dist = 0
                d_have = cal_distance(p1, p2)
                d_need = each_dist - dist

            dist += d_have

        return pd.DataFrame({'lon': lon, 'lat': lat, 'dates': dates})

    def plot_obs(self, fill_num=None, one_by_one=False):
        if fill_num is None:
            data = self.obs
        else:
            data = self.fill_with_dist(fill_num)
        fig = plt.figure()
        # plt.show()
        ax = fig.add_subplot(1, 1, 1)
        plt.grid(True)  # 添加网格
        plt.ion()  # interactive mode on
        if one_by_one:
            for i in range(len(data)):
                ax.plot(data.lon[:i], data.lat[:i])
                plt.pause(0.2)
        else:
            ax.plot(data.lon, data.lat)
        plt.pause(2)

    def cal_time_interval(self):
        total_time =  (max(self.obs.dates) - min(self.obs.dates)).total_seconds()
        return total_time / (len(self.obs) - 1)

def multi_map_matching(obses, graph, save_name,
                       mdtw_interval=5,
                       mm_smoothing=10,
                       mm_interpolate_interval=10,
                       save_dir='../../res/work/0417_multi_mm/'):
    mdtw = MultiDTWIndx(obses, interval=mdtw_interval)
    new_obs = mdtw.run_one_by_one()
    print(new_obs)
    output_data_js(new_obs,  save_dir+save_name+'_raw.js')
    track = CTrack(new_obs, graph, smoothing=mm_smoothing, interpolate_interval=mm_interpolate_interval)
    output_data_js(track.obs, save_dir+save_name+'_processed.js')
    states = track.viterbi()
    output_data_js(states, save_dir+save_name+'.js')
    return states

def single_map_matching(obs, graph, save_name,
                        mm_smoothing=10,
                        mm_interpolate_interval=10,
                        save_dir='../../res/work/0417_multi_mm/'):
    output_data_js(obs,  save_dir+save_name+'_raw.js')
    track = CTrack(obs, graph, smoothing=mm_smoothing, interpolate_interval=mm_interpolate_interval)
    output_data_js(track.obs, save_dir+save_name+'_processed.js')
    states = track.viterbi()
    output_data_js(states, save_dir+save_name+'.js')
    return states

def dist_trajectories(ts1, ts2):
    assert len(ts2) == len(ts1)
    dist = 0
    for i in range(len(ts1)):
        dist += cal_distance(ts1[i], ts2[i])
    return dist / len(ts1)

def test_observation(trail, direction, n):
    '''
    Test the class observation.
    :param trail: The number of trail.
    :param direction: [0/1]
                      If direction == 1: positive direction
                      else: negative direction
    :param n: The number of test.
    :return:
    '''
    obs = Observation(get_test_obs(trail, direction, n))
    print(obs.obs)
    print('-'*10)
    print(obs.fill_with_dist(5))
    print(obs.get_total_dist())
    obs.plot_obs(3)

def test_observations(trail, direction, fill_num):
    '''
    Test observation list.
    :param trail:
    :param direction:
    :param fill_num:
    :return:
    '''
    obses = []
    for n in range(1, 11):
        obses.append(get_test_obs(trail, direction, n))
    obses = Observations(obses)
    # obses.plot_obs(fill_num)
    new_obses = obses.get_mean_obs(fill_num)
    Observation(new_obses).plot_obs()
    print(new_obses)

def test_dtw(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need[:2]:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obs = interpolate(obs, interval=1)
        obs = [[obs.lon[i], obs.lat[i]] for i in range(len(obs))]
        obses.append(obs)

    dtw = DTW(obses[0], obses[1], is_norm=False)
    mat, mat_ind = dtw.shortest_path_dtw(is_derivatives=True)
    # mat, mat_ind = dtw.quick_dtw(is_derivatives=False)
    # mat, mat_ind = dtw.dtw()
    trace, trace_mat = dtw.get_trace(mat_ind)
    dtw.plot(trace, 'random_user%d_%d_%d DDTW Plot' % (user_number, path_need[0], path_need[1]))
    # ts1, ts2 = dtw.get_new_ts()
    # print(dist_trajectories(ts1, ts2))
    # return trace

def test_muti_dtw(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obs = interpolate(obs, interval=60)
        obs = [[obs.lon[i], obs.lat[i]] for i in range(len(obs))]
        obses.append(obs)

    dtw = MultiDTW(obses)
    # mat, mat_ind = dtw.dtw_tses()
    dtw.plot()
    # mat, mat_ind = dtw.quick_dtw(is_derivatives=False)
    # mat, mat_ind = dtw.dtw()
    # trace, trace_mat = dtw.get_trace(mat_ind)
    # dtw.plot(trace, 'random_user%d_%d_%d DDTW Plot' % (user_number, path_need[0], path_need[1]))
    # ts1, ts2 = dtw.get_new_ts()
    # print(dist_trajectories(ts1, ts2))
    # return trace

def test_hefei(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        print(Observation(obs).cal_time_interval())
        obses.append(obs)

    obses = Observations(obses)
    # obses.plot_obs()
    obses.save_json('../../res/work/0315_similar_trajectories/random_user%d.json' % user_number,
                    data['user_id'], is_json=True)
    obses.save_json('../../res/work/0315_similar_trajectories/random_user%d.js' % user_number,
                    data['user_id'], is_json=False, fill_num=30)

    return obses

def obs_to_ts(obs, interval=1):
    '''
    Transform observation to ts
    :param obs: DataFrame['lon', 'lat', 'dates']
    :param interval: The interval to interpolate the observation
    :var ts: [[lon, lat]]
    :return: ts
    '''
    obs = interpolate(obs, interval=interval)
    ts = []
    for t in range(len(obs)):
        ts.append([obs.lon[t], obs.lat[t]])

    return ts

def test_multi_mm(user_number, path_need, smoothing=2, interpolate_interval=15, interpolate=True):
    '''
    Test Multi_mm class.
    :param user_number: Random User number use
    :param path_need: Path number used
    :param smoothing:
    :param interpolate_interval:
    :param interpolate:
    :return:
    '''

    # Get observations
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obses.append(obs)

    # Get graph
    graph = get_graph('../../data/hefei_road/link_baidu.txt')

    matched_obses = []
    mean_obs = Observations(obses).get_mean_obs(30)
    obses.append(mean_obs)
    for obs in obses:
        track = CTrack(obs, g=graph, smoothing=smoothing,
                       interpolate_interval=interpolate_interval,
                       interpolate=interpolate)
        matched_obs = track.viterbi()
        matched_obses.append(matched_obs)

    trajectories_tojson(matched_obses, data['user_id'],
                        '../../res/work/0315_similar_trajectories/multiHMM_random_user%d.js' % user_number,
                        is_json=False)
    trajectories_tojson(matched_obses, data['user_id'],
                        '../../res/work/0315_similar_trajectories/multiHMM_random_user%d.json' % user_number,
                        is_json=True)

def test_multi_index(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obses.append(obs)

    mdtw = MultiDTWIndx(obses, interval=5)
    new_obs = mdtw.run_one_by_one()
    mdtw.plot(new_obs)
    obses.append(new_obs)
    trajectories_tojson(obses, data['user_id'],
                        '../../res/work/0411_dtw_index_sort/multiHMM_random_user%d.js' % user_number,
                        is_json=False)

# dijkstra算法实现，有向图和路由的源点作为函数的输入，最短路径最为输出
def dijkstra(graph, src=0):
    # 判断图是否为空，如果为空直接退出
    if graph is None:
        return None
    nodes = [i for i in range(len(graph))]  # 获取图中所有节点
    visited=[]  # 表示已经路由到最短路径的节点集合
    if src in nodes:
        visited.append(src)
        nodes.remove(src)
    else:
        return None
    distance={src:0}  # 记录源节点到各个节点的距离
    for i in nodes:
        distance[i]=graph[src][i]  # 初始化
    # print(distance)
    path={src:{src:[]}}  # 记录源节点到每个节点的路径
    k=pre=src
    while nodes:
        mid_distance=float('inf')
        for v in visited:
            for d in nodes:
                new_distance = graph[src][v]+graph[v][d]
                if new_distance < mid_distance:
                    mid_distance=new_distance
                    graph[src][d]=new_distance  # 进行距离更新
                    k=d
                    pre=v
        distance[k]=mid_distance  # 最短路径
        path[src][k]=[i for i in path[src][pre]]
        path[src][k].append(k)
        # 更新两个节点集合
        visited.append(k)
        nodes.remove(k)
        print(visited,nodes)  # 输出节点的添加过程
    return distance,path

def cal_avg_time_interval():
    users = []
    num = []
    intervals = []
    for user_number in range(8, 63):
        with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
            data = json.load(data_file)
        for i in range(len(data['trajectories'])):
            tmp = data['trajectories'][i]
            dates = [get_datetime(t) for t in tmp['dates']]
            obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})

            users.append(user_number)
            num.append(i)
            intervals.append(Observation(obs).cal_time_interval())
        out = pd.DataFrame({'user': users, 'num': num, 'time_interval': intervals})
        out.to_csv('../../res/work/0506_big_eye/time_interval/time_interval.csv')

    import seaborn as sns
    sns.set_palette('deep', desat=.6)
    sns.set_context(rc={'figure.figsize': (8, 5)})
    plt.hist(intervals, bins=20)
    plt.grid(True, linestyle="--", color="black", linewidth=".5", alpha=.4)
    plt.title('Cellular Data Time Interval Histogram')
    plt.xlabel('Time interval(s)')
    plt.ylabel('Frequency')
    plt.show()

if __name__ == '__main__':
    # test_observation(1,1,1)
    # test_observations(1, 1, 10)
    # test_hefei(8, [15, 25, 26])
    # test_multi_mm(8, [15, 25, 26])
    # test_dtw(8, [15, 25])
    # print(len(get_ksum(10, 3)))
    # test_muti_dtw(8, [15, 25, 26])
    # test_multi_index(8, [15, 25, 26])
    # test_multi_index(20, [15, 38, 2, 7, 19])
    cal_avg_time_interval()

    # graph_list = [[0, 2, 1, 4, 5, 1],
    #               [1, 0, 4, 2, 3, 4],
    #               [2, 1, 0, 1, 2, 4],
    #               [3, 5, 2, 0, 3, 3],
    #               [2, 4, 3, 4, 0, 1],
    #               [3, 4, 7, 3, 1, 0]]
    #
    # distance, path = dijkstra(graph_list, 0)  # 查找从源点0开始带其他节点的最短路径
    # print(distance, path)


import weakref
import importlib

class Component(object):
    __slots__ = [
        '_entity',
        'typeid',
        'synchronize',
        '_is_unique',
        '_import_string',
    ]

    def __init__(self):
        self._is_unique = False
        self.synchronize = False
        self._import_string = self.__class__.__module__ + '.' + self.__class__.__name__

    def __del__(self):
        #print("__del__", self)
        pass

    def cleanup(self):
        pass

    @property
    def entity(self):
        return self._entity()

    @property
    def is_unique(self):
        return self._is_unique

    def serialize(self):
        return {'import_string': self._import_string}

class UniqueComponent(Component):
    __slots__ = []

    def __init__(self):
        super().__init__()
        self._is_unique = True

class Entity(object):
    __slots__ = [
        '_components',
        '_new_components',
        '__weakref__',
        'guid',
        'space',
        'netid',
    ]

    def __init__(self, space):
        self._components = {}
        self._new_components = {}
        self.guid = None
        self.space = space
        self.netid = 0

    def __del__(self):
        for k, v in self._components.items():
            for i in v:
                i.cleanup()
        self._components.clear()

        for k, v in self._new_components.items():
            for i in v:
                i.cleanup()
        self._new_components.clear()
        #print("__del__", self)

    def add_component(self, component):
        typeid = component.typeid

        enforce_unique = component.is_unique
        if not enforce_unique:
            try:
                enforce_unique = bool([i for i in self.get_components(typeid) if i.is_unique])
            except KeyError:
                enforce_unique = False

        if enforce_unique and self.has_component(typeid):
            raise RuntimeError('Entity already has component with typeid of {}'.format(typeid))
        component._entity = weakref.ref(self)

        if typeid in self._new_components:
            self._new_components[typeid].append(component)
        else:
            self._new_components[typeid] = [component]

    def remove_component(self, component):
        if component.typeid in self._components:
            d = self._components
            clist = self._components[component.typeid]
        elif component.typeid in self._new_components:
            d = self._new_components
            clist = self._new_components[component.typeid]
        else:
            raise KeyError('Enity has no component with typeid of {}'.format(component.typeid))

        component.cleanup()
        clist.remove(component)
        if not clist:
            del d[component.typeid]

    def get_component(self, typeid):
        if len(self.get_components(typeid)) > 1:
            raise RuntimeError('Entity has more than one component with typeid of {}'.format(typeid))

        return self.get_components(typeid)[0]

    def get_components(self, typeid):
        if typeid in self._components:
            return self._components[typeid]
        elif typeid in self._new_components:
            return self._new_components[typeid]
        else:
            raise KeyError('Enity has no component with typeid of {}'.format(typeid))

    def has_component(self, typeid):
        return typeid in self._components or typeid in self._new_components

    def serialize(self):
        d = {}
        for typeid, clist in self._components.items():
            d[typeid] = []
            for component in clist:
                if component.synchronize:
                    d[typeid].append(component.serialize())
            if not d[typeid]:
                del d[typeid]

        return d

    def update(self, netid, d):
        self.netid = netid
        #print(self.netid, d)
        for typeid, clist in d.items():
            for i, cdata in enumerate(clist):
                mod_parts = cdata['import_string'].split('.')
                cmod = '.'.join(mod_parts[:-1])
                cclass = mod_parts[-1]

                try:
                    component = self.get_components(typeid)[i]
                except (IndexError, KeyError):
                    mod = importlib.import_module(cmod)
                    component = getattr(mod, cclass)()
                    self.add_component(component)

                component.update(cdata)

class System(object):
    __slots__ = [
        'component_types',
    ]

    def init_components(self, dt, entities):
        pass

    def update(self, dt, entities):
        pass

class DuplicateSystemException(Exception):
    pass

class ECSManager(object):
    def __init__(self):
        self.entities = []
        self.systems = {}
        self.next_entity_guid = 0
        self.space = Entity(None)
        self.removed_entities = set()

    def create_entity(self):
        # TODO allow for multiple spaces
        if self.space is None:
            raise RuntimeError("ECSManager.space is None")
        entity = Entity(self.space)
        self._add_entity(entity)

        return entity

    def _add_entity(self, entity):
        entity.guid = self.next_entity_guid
        self.next_entity_guid += 1
        self.entities.append(entity)

    def remove_entity(self, entity):
        if entity.netid != 0:
            self.removed_entities.add(entity.netid)
        self.entities.remove(entity)

    def add_system(self, system):
        name = system.__class__.__name__
        if name in self.systems:
            raise DuplicateSystemException("{} has already been added.".format(name))
        self.systems[name] = system

    def has_system(self, system_str):
        return system_str in self.systems

    def get_system(self, system_str):
        if system_str not in self.systems:
            raise KeyError('No system found with the name of {}'.format(system_str))
        return self.systems[system_str]

    def remove_system(self, system_str):
        if system_str not in self.systems:
            raise KeyError('No system found with the name of {}'.format(system_str))
        del self.systems[system_str]

    def remove_space(self):
        # TODO allow for multiple spaces
        for entity in self.entities[:]:
            if entity.space == self.space:
                self.entities.remove(entity)

        self.space = None

    def _get_components_by_type(self, component_list, component_types):
        components = {k: [] for k in component_types}
        for entity in self.entities:
            for typeid in component_types:
                components[typeid] += getattr(entity, component_list).get(typeid, [])

        return components

    def update(self, dt):
        entities = self.entities[:]

        for system in self.systems.values():
            system.init_components(dt, self._get_components_by_type('_new_components', system.component_types))

        for entity in entities:
            for typeid, clist in entity._new_components.items():
                if typeid in entity._components:
                    entity._components[typeid].extend(clist)
                else:
                    entity._components[typeid] = clist[:]
            entity._new_components.clear()

        for system in self.systems.values():
            system.update(dt, self._get_components_by_type('_components', system.component_types))


import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
# local model
import sys
sys.path.append("../network")
from lstm import LSTMHardSigmoid

class cnnblstm(nn.Module):

	def __init__(self, time_steps = 800, n_features = 3, n_outputs = 10, params_file = "./params.pkl", use_cuda = False):
		super(cnnblstm, self).__init__()

		self.time_steps = time_steps
		self.n_features = n_features
		self.n_outputs = n_outputs
		self.params_file = params_file
		self.use_cuda = use_cuda

		self.n_filters = 128
		self.kernel_size = 15
		self.n_hidden = 150	# 150
		self.n_layers = 1
		self.bidirectional = True

		# build net1 cnn
		self.net1 = nn.Sequential(
			nn.Conv1d(in_channels = self.n_features, out_channels = self.n_filters, kernel_size = self.kernel_size),
			nn.ReLU(),
			# nn.Sigmoid(),
			nn.Dropout(p = 0.5),
			nn.MaxPool1d(kernel_size = 2)
		)

		# build net2 features.blstm
		# self.net2 = nn.LSTM(input_size = 128, hidden_size = self.n_hidden, num_layers = self.n_layers, dropout = 0.2, batch_first = True, bidirectional = self.bidirectional, bias = True)
		self.net2 = LSTMHardSigmoid(input_size = self.n_filters, hidden_size = self.n_hidden, num_layers = self.n_layers, dropout = 0.2, batch_first = True, bidirectional = self.bidirectional, bias = True)
		if self.bidirectional:
			n_blstm_output = self.n_hidden * 2
		else:
			n_blstm_output = self.n_hidden

		# build net3 classifier(fc->fc)
		self.net3 = nn.Sequential(
			# nn.Tanh(),
			nn.Linear(n_blstm_output, 50, bias = True),
			nn.ReLU(),
			# nn.Sigmoid(),
			nn.Dropout(p = 0.2),
			nn.Linear(50, self.n_outputs, bias = True),
			nn.Softmax(dim = 1)
		)

	def reset_parameters(self):
		"""
		temp useless
		Here we reproduce Keras default initialization weights for consistency with Keras version
		"""
		# get weights & bias set
		net1_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net1" in name) and ("net1_adabn" not in name))))
		net1_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net1" in name) and ("net1_adabn" not in name))))
		# net2_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net2" in name) and ("net2_adabn" not in name))))
		# net2_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net2" in name) and ("net2_adabn" not in name))))
		net3_weights = ((name, param.data) for name, param in self.named_parameters() if (("weight" in name) and (("net3" in name) and ("net3_adabn" not in name))))
		net3_biases = ((name, param.data) for name, param in self.named_parameters() if (("bias" in name) and (("net3" in name) and ("net3_adabn" not in name))))
		# init weights & bias
		for name, params_data in net1_weights:
			nn.init.xavier_uniform_(params_data)
		for name, params_data in net1_biases:
			nn.init.constant_(params_data, 0)
		self.net2.reset_parameters()		# lstm reset parameters
		for name, params_data in net3_weights:
			nn.init.xavier_uniform_(params_data)
		for name, params_data in net3_biases:
			nn.init.constant_(params_data, 0)

	def forward(self, input):
		"""
		compute the output of input according to the entire network model
		"""
		# MaxPool1d
		maxPool1d_output = self.net1(input)
		maxPool1d_t_output = maxPool1d_output.permute(0, 2, 1).contiguous()
		# BiLSTM
		bilstm_output, _ = self.net2(maxPool1d_t_output, None)
		# MaxPooling1D time_steps
		# print(bilstm_output.size())
		bilstm_output = bilstm_output.permute(0, 2, 1)
		# print(bilstm_output.size())
		maxPooling_output = F.max_pool1d(bilstm_output, kernel_size = bilstm_output.size(2)).squeeze(2)
		# maxPooling_output = bilstm_output[:, :, -1].squeeze(-1)
		# print(maxPooling_output.size())
		# get classifier
		linear2_softmax_output = self.net3(maxPooling_output)

		return linear2_softmax_output

	def trainAllLayers(self, train_x, train_y, learning_rate = 0.001, n_epoches = 20, batch_size = 10, shuffle = True):
		"""
		train all layers of network model
		"""
		# get train_data
		train_data = torch.utils.data.TensorDataset(train_x, train_y)
		# Data Loader for easy mini-batch return in training
		train_loader = torch.utils.data.DataLoader(dataset = train_data, batch_size = batch_size, shuffle = shuffle)
		# optimize all cnn parameters
		optimizer = torch.optim.Adam(self.parameters(), lr = learning_rate)
		# the target label is not one-hotted
		loss_func = nn.CrossEntropyLoss()

		# init params
		self.reset_parameters()

		# load params
		self.load_params()

		# set train mode True
		self.train()

		# training and testing
		for epoch in range(n_epoches):
			# init loss & acc
			train_loss = 0
			train_acc = 0
			for step, (b_x, b_y) in enumerate(train_loader):		# gives batch data
				b_x = b_x.view(-1, self.n_features, self.time_steps)	# reshape x to (batch, n_features, time_step)
				if self.use_cuda:
					b_x, b_y = Variable(b_x).cuda(), Variable(b_y).cuda()
				else:
					b_x, b_y = Variable(b_x), Variable(b_y)
				# get output
				output = self(b_x)									# CNN_BLSTM output
				# get loss
				loss = loss_func(output, b_y)						# cross entropy loss
				train_loss += loss.item() * len(b_y)
				_, pre = torch.max(output, 1)
				num_acc = (pre == b_y).sum()
				train_acc += num_acc.item()
				# backward
				optimizer.zero_grad()								# clear gradients for this training step
				loss.backward()										# backpropagation, compute gradients
				optimizer.step()									# apply gradients

				# print loss
				# if (step + 1) % 10 == 0:
				# 	print("[{}/{}], train loss is: {:.6f}, train acc is: {:.6f}".format(step, len(train_loader), train_loss / ((step + 1) * batch_size), train_acc / ((step + 1) * batch_size)))
			print("[{}/{}], train loss is: {:.6f}, train acc is: {:.6f}".format(len(train_loader), len(train_loader), train_loss / (len(train_loader) * batch_size), train_acc / (len(train_loader) * batch_size)))

			# save params
			self.save_params()

		print("train finish!")

	def getTestAccuracy(self, test_x, test_y):
		"""
		test network model with test set
		"""
		# init params
		self.reset_parameters()

		# load params
		self.load_params()

		# set eval
		self.eval()

		with torch.no_grad():
			if self.use_cuda:
				test_x, test_y = Variable(test_x).cuda(), Variable(test_y).cuda()
			else:
				test_x, test_y = Variable(test_x), Variable(test_y)
		# get output
		output = self(test_x)
		# print(output)
		prediction = torch.max(output, 1)[1]
		pred_y = prediction.cpu().data.numpy()
		# print(pred_y)
		target_y = test_y.cpu().data.numpy()
		# print(test_y)

		accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
		print("Accuracy: ", str(accuracy))

	def save_params(self):
		"""
		save params
		"""
		torch.save(self.state_dict(), self.params_file)
		print("save_params success!")

	def load_params(self):
		"""
		load params
		"""
		if os.path.exists(self.params_file):
			if self.use_cuda:
				self.load_state_dict(torch.load(self.params_file, map_location = torch.device('cuda')))
			else:
				self.load_state_dict(torch.load(self.params_file, map_location = torch.device('cpu')))
			print("load_params success!")

	def get_model(self, pre_trained = False):
		"""
		get pretrained model
		"""
		if pre_trained:
			self.load_params()
		return self

if __name__ == '__main__':
	use_cuda = torch.cuda.is_available()
	if use_cuda:
		cnnblstm = cnnblstm(use_cuda = use_cuda).cuda()
	else:
		cnnblstm = cnnblstm(use_cuda = use_cuda)
	print(cnnblstm)
	# get train_x, train_y
	train_x = torch.rand(20, 3, 800, dtype = torch.float32)
	train_y = torch.randint(10, (20, ), dtype = torch.int64)
	# train_y = torch.LongTensor(20, 1).random_() % 10
	print(train_x.type())
	# train_y = torch.zeros(20, 10).scatter_(1, train_y, 1)
	print(train_y)
	train_data = torch.utils.data.TensorDataset(train_x, train_y)
	cnnblstm.trainAllLayers(train_data)


import openai as ai
completion = ai.Completion()

def chatbot_interface(description, history, sentence, api_key):
    ai.api_key = api_key
    chat_log = description + "\n" + history + "\n"
    prompt = f"{chat_log}Human: {sentence}\nAI:"
    response = completion.create(
        model="text-davinci-002",
        prompt=prompt,
        temperature=0.85,
        max_tokens=100,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0.6,
        stop=[" Human:", " AI:"])
    res = response.choices[0].text.strip('\n')

    return res

if __name__ == "__main__":
    history = ""

    #description = "This is a conversation with an AI assistant. This assistant is very kind, helpful and humorous."
    description = "your-description"

    #user_sentences = ["Hi, do you know the Avengers?", "What's your favourite character of the Avengers?"]
    user_sentences = "your-sentences"

    api_key = "your-api-key"

    #an example of usage, the user_sentences can be obtained from other interface(e.g. front-end of a softerware, user input)
    #while fit your conditions
    for i in range(2):
        res = chatbot_interface(description, history, user_sentences[i], api_key)
        print(f"user: {user_sentences[i]}\nchatbot: {res}")
        user_history = "Human: " + user_sentences[i] + "\n"
        chatbot_history = "AI: " + res
        history = history + user_history + chatbot_history


import os
import tempfile

import numpy as np
import numpy.testing as npt
import pytest
from pygt3x.reader import FileReader

from paat import io, preprocessing

@pytest.fixture
def unscaled_data(file_path):
    return io.read_gt3x(file_path, rescale=False, pandas=True)

def test_loading_data(file_path, load_gt3x_file):
    data, _ = load_gt3x_file
    _, acceleration, _ = io.read_gt3x(file_path, rescale=True, pandas=False)

    assert np.array_equal(acceleration, data[["Y", "X", "Z"]].values)

@pytest.mark.slow
def test_against_actigraph_implementation(file_path, unscaled_data):
    with FileReader(file_path) as reader:
        ref = reader.to_pandas()

    npt.assert_almost_equal(unscaled_data[["X", "Y", "Z"]].values, ref[["X", "Y", "Z"]].values)
    #assert np.allclose(data[["X", "Y", "Z"]].values, ref[["X", "Y", "Z"]].values)

def test_exceptions():
    with pytest.raises(NotImplementedError) as e_info:
        time_data = np.arange(10)
        hz = 33
        io._create_time_array(time_data, hz)
        assert e_info

    with pytest.raises(NotImplementedError) as e_info:
        start = np.asarray(np.datetime64('today'), dtype='datetime64[ms]')
        n_samples = 330
        hz = 33
        io._create_time_vector(start, n_samples, hz)
        assert e_info


import dataclasses
import importlib
import logging
from typing import Dict
from typing import Iterable
from typing import Optional
from typing import Tuple
from typing import Union

import OpenGL.GL
import OpenGL.GL.NV.conservative_raster
import glcontext
import moderngl
import numpy as np
import torch as t

from corenet.gl import egl_context

importlib.reload(glcontext)
egl_context.monkey_patch_moderngl()

InputTensor = Union[t.Tensor, np.ndarray, int, float, bool, Iterable]

log = logging.getLogger(__name__)

@dataclasses.dataclass(frozen=True)
class Uniform:
  name: str
  value: InputTensor

@dataclasses.dataclass()
class Buffer:
  binding: int
  value: InputTensor
  is_io: bool = False

@dataclasses.dataclass(frozen=True)
class Texture:
  name: str
  value: InputTensor
  bind_as_array: bool = False

@dataclasses.dataclass(frozen=True)
class RenderInput:
  # The number of points to render. The geometry shader can then convert these
  # into other geometry.
  num_points: int

  # The parameters to pass to the shaders.
  arguments: Iterable[Union[Uniform, Buffer, Texture]]

  # The vertex shader
  vertex_shader: str

  # The geometry shader
  geometry_shader: str

  # The fragment shader
  fragment_shader: str

  # The output resolution, tuple (height, width)
  output_resolution: Tuple[int, int] = (256, 256)

  # The clear color, tuple (R, G, B) or (R, G, B, A).
  clear_color: Iterable[float] = (0, 0, 0)

  # The output type (either uint8 or float32).
  output_type: t.dtype = t.uint8

  # Whether depth testing is enabled.
  depth_test_enabled: bool = True

  # Whether to use conservative rasterization
  conservative_rasterization: bool = False

class _EglRenderer:
  _context_cache = {}  # type: Dict[int, _EglRenderer]

  def __init__(self, gl_context: moderngl.Context):
    self._program_cache = {}  # type: Dict[str, moderngl.Program]
    self._gl_context = gl_context

  @classmethod
  def get_instance(cls, cuda_device: int):
    if cuda_device not in cls._context_cache:
      # Cuda has to be initialized for the cuda_egl backend to work
      t.cuda.init()
      ctx = moderngl.create_standalone_context(backend="cuda_egl",
                                               cuda_device=cuda_device)
      cls._context_cache[cuda_device] = _EglRenderer(ctx)
    return cls._context_cache[cuda_device]

  def _check_gl_error(self):
    err = self._gl_context.error
    if err != "GL_NO_ERROR":
      raise ValueError(f"OpenGL error encountered: {err}")

  def _upload_uniform(self, program: moderngl.Program,
                      parameter: Uniform):
    if parameter.name not in program:
      log.info(f"Uniform {parameter.name} not found in program")
    else:
      value = t.as_tensor(parameter.value).cpu()
      if len(value.shape) == 2:
        value = tuple(value.transpose(0, 1).reshape([-1]))
      elif len(value.shape) == 1:
        value = tuple(value)
      elif value.shape == ():
        value = value.item()
      else:
        raise ValueError("Only supports 0, 1, and 2 dim tensors.")
      program[parameter.name].value = value

  def render(self, render_input: RenderInput):
    inp = render_input
    with self._gl_context:
      program_key = (
          f"{inp.vertex_shader}|{inp.geometry_shader}|{inp.fragment_shader}")

      if program_key not in self._program_cache:
        self._program_cache[program_key] = self._gl_context.program(
            vertex_shader=inp.vertex_shader,
            fragment_shader=inp.fragment_shader,
            geometry_shader=inp.geometry_shader)
      program = self._program_cache[program_key]

      objects_to_delete = []
      buffer_bindings = {}  # type: Dict[int, moderngl.Buffer]
      texture_location = 0
      try:
        for parameter in inp.arguments:
          if isinstance(parameter, Uniform):
            self._upload_uniform(program, parameter)
          elif isinstance(parameter, Buffer):
            value = t.as_tensor(parameter.value).reshape([-1]).cpu().numpy()
            buffer = self._gl_context.buffer(value)
            buffer.bind_to_storage_buffer(parameter.binding)
            buffer_bindings[parameter.binding] = buffer
            objects_to_delete.append(buffer)
          elif isinstance(parameter, Texture):
            if not parameter.bind_as_array:
              raise NotImplementedError()
            if parameter.name not in program:
              log.info(f"Uniform {parameter.name} not found in program")
            else:
              val = parameter.value.cpu().numpy()
              texture = self._gl_context.texture_array(
                  val.shape[1:3] + val.shape[0:1], val.shape[3], val)
              texture.repeat_x = True
              texture.repeat_y = True
              texture.use(location=texture_location)
              program.get(parameter.name, None).value = texture_location
              texture_location += 1
              objects_to_delete.append(texture)
          else:
            raise ValueError("Unknown parameter type")
          self._check_gl_error()

        h, w = inp.output_resolution
        gl_dtype, np_dtype = {
            t.uint8: ("f1", np.uint8),
            t.float32: ("f4", np.float32)
        }[inp.output_type]

        render_buffer = self._gl_context.renderbuffer(
            (w, h), components=4, samples=0, dtype=gl_dtype)
        objects_to_delete.append(render_buffer)
        depth_buffer = self._gl_context.depth_renderbuffer((w, h), samples=0)
        objects_to_delete.append(depth_buffer)
        framebuffer = self._gl_context.framebuffer(render_buffer, depth_buffer)
        objects_to_delete.append(framebuffer)
        framebuffer.use()

        vertex_array = self._gl_context.vertex_array(program, ())
        objects_to_delete.append(vertex_array)
        self._gl_context.clear(*inp.clear_color)
        self._gl_context.disable(moderngl.CULL_FACE)
        if inp.depth_test_enabled:
          self._gl_context.enable(moderngl.DEPTH_TEST)
          self._gl_context.depth_func = "<="
        else:
          self._gl_context.disable(moderngl.DEPTH_TEST)

        if render_input.conservative_rasterization:
          OpenGL.GL.glEnable(
              OpenGL.GL.NV.conservative_raster.GL_CONSERVATIVE_RASTERIZATION_NV)
        else:
          OpenGL.GL.glDisable(
              OpenGL.GL.NV.conservative_raster.GL_CONSERVATIVE_RASTERIZATION_NV)

        vertex_array.render(mode=moderngl.POINTS, vertices=inp.num_points)
        self._check_gl_error()
        result = np.zeros([h, w, 4], dtype=np_dtype)
        framebuffer.read_into(result, components=4, dtype=gl_dtype)
        self._check_gl_error()

        for parameter in inp.arguments:
          if isinstance(parameter, Buffer) and parameter.is_io:
            old = parameter.value
            npdt = {t.float32: np.float32, t.int32: np.int32}[old.dtype]
            temp_buffer = np.zeros(old.shape, npdt)
            buffer_bindings[parameter.binding].read_into(temp_buffer)
            parameter.value = t.as_tensor(temp_buffer)

        return t.as_tensor(result)

      finally:
        for v in objects_to_delete:
          v.release()

def gl_simple_render(
    render_input: RenderInput,
    cuda_device: Optional[int] = None
):
  """Renders the supplied configuration with OpenGL.

  Args:
    render_input: The render input
    cuda_device: The GPU to use, given as a CUDA device number

  Returns:
    The rendered image, output_type[height, width, 4]

  Buffer parameters with is_io set to True will be read back. This can either
  happen in the original buffer or in a new buffer. The implementation will
  update the value field in the buffer parameter in the latter case.

  There is no guarantee about the device of both the rendered image and the
  buffers that are read back.
  """
  if cuda_device is None:
    cuda_device = t.cuda.current_device()
  instance = _EglRenderer.get_instance(cuda_device)
  return instance.render(render_input)


"""
Created on Sat Mar  7 08:48:27 2020

@author: SamKoebrich
"""

import os
from pathlib import Path

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ~~~~~~~~~~~ GENERAL SETTINGS ~~~~~~~~~~~~~
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# --- Number of generators for training set ---
#   - most downloads are cached, so if you set a higher number,
#     and then a lower, you don't lose data.
RUN_PRE_EE = True
FETCH_S3 = True
RUN_BAYES_OPT = True
N_GENERATORS = None
MAX_YEAR = 2019
SCHEMA = 'production'

# --- Multiprocessing settings ---
MULTIPROCESSING = True
WORKERS = 12
THREADS = 6 #ThreadPoolExecutor is failing for Earth Engine queries, so this is still using ProcessPool

# --- Bool for detailed output ---
VERBOSE = False

# --- Frequency of observations (i.e. 'D' for daily, 'W', for weekly, 'M' for monthly, 'A' for annual) ---
TS_FREQUENCY = '1W'
if TS_FREQUENCY in ['1W']:
    TS_DIVISOR = 52
elif TS_FREQUENCY in ['MS']:
    TS_DIVISOR = 12
elif TS_FREQUENCY in ['D', '1D']:
    TS_DIVISOR = 365
elif TS_FREQUENCY in ['A','AS']:
    TS_DIVISOR = 1
elif TS_FREQUENCY == '3D':
    TS_DIVISOR = int(365 / 3)
elif TS_FREQUENCY == '2D':
    TS_DIVISOR = int(365 / 2)
else:
    raise NotImplementedError(f'Please write a wrapper for {TS_FREQUENCY}!')

TRAIN_COUNTRIES = ['United States of America']
PREDICT_COUNTRIES = ['WORLD'] #['Colombia', 'Venezuela', 'Costa Rica', 'Guatemala', 'Jamaica', 'Puerto Rico']

PREDICT_START_DATE = '01-01-2019'
PREDICT_END_DATE = '12-31-2019'

CEMS_MEASUREMENT_FLAGS = [] #blank for include everything

FETCH_SENTINEL_FROM = 'S3'

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ~~~~~~~~~ EARTH ENGINE SETTINGS ~~~~~~~~~~
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# --- Sources ---
EARTHENGINE_DBS = [
    "CIESIN/GPWv411/GPW_Population_Count",
    "ECMWF/ERA5/DAILY"
]

# --- API Query settings ---
EE_TIMEOUT = 120 #forced timeout, overriding exponential backoff before calling a API query nans
RETRY_EE_NANS = False #after loading cache, retry queries that returned nans in last call

# --- Scale (in meters) to query ---
BUFFERS = [20000] #1e2, 1e4

# --- Hour to grab after --- (i.e. don't consider an observation unless it is after this hour)
LEFT_WINDOW_HOUR = 20

# --- Degrees to match GPPD and EIA dataframes ---
DEGREES_DISTANCE_MATCH = 0.005

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ~~~~~~~~~~~~~~ L3 SETTINGS ~~~~~~~~~~~~~~~
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

RAW_S3_DIR = os.path.join(str(Path.home()),'HDD','tycho_raw')

S3_DBS = [
        'COGT/OFFL/L2__SO2___/',
        # 'COGT/OFFL/L2__O3____/', #available 2019-04 on
        'COGT/OFFL/L2__NO2___/',
        'COGT/OFFL/L2__HCHO__/',
        'COGT/OFFL/L2__CH4___/',
        'COGT/OFFL/L2__CO____/',
        # 'COGT/OFFL/L2__CLOUD_/' #available 2019-04 on
        'COGT/OFFL/L2__AER_AI/'
        ]

S3_BUCKET = 'meeo-s5p'

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ~~~~~~~ SANITIZE/SPLIT SETTINGS ~~~~~~~~~~
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# --- US y (target) variables ---
CEMS_Y_COLS = [ #LEAKAGE WARNING! Removing something from this list will cause it to persist in the X sets
    'gross_load_mw',
    'so2_lbs',
    'nox_lbs',
    'operational_time',
    'co2_lbs'
]

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ~~~~~~~~~~~~~ ML SETTINGS ~~~~~~~~~~~~~~~~
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
HOW_TO_SPLIT = 0.1 #test frac
TRAIN_MODEL = 'bayes-xgb'
PREDICT_MODEL = 'bayes-xgb'
ML_Y_COLS = ['gross_load_mw','so2_lbs','nox_lbs','co2_lbs']
CV_FOLDS = 3

RANDOMSEARCH_ITER = 50

# --- Params for Bayes training ---
BAYES_N_ITER = 50
BAYES_INIT_POINTS = 100
BAYES_ACQ = 'ucb'

# --- Params for TPOT Training ---
TPOT_GENERATIONS = 100
TPOT_POPULATION_SIZE = 100
TPOT_TIMEOUT_MINS = 60*24
# TPOT_CONFIG_DICT = tpot_configs.lgbm_config_dict


import base64
import binascii
import struct
from typing import Union

from crc16 import crc16xmodem

from .exceptions import (
    Ed25519SecretSeedInvalidError,
    Ed25519PublicKeyInvalidError,
    MuxedEd25519AccountInvalidError,
    ValueError,
    TypeError,
)
from .xdr import Xdr

__all__ = ["StrKey"]

_version_bytes = {
    "ed25519_public_key": binascii.a2b_hex("30"),  # G 48 6 << 3
    "ed25519_secret_seed": binascii.a2b_hex("90"),  # S 144 18 << 3
    "pre_auth_tx": binascii.a2b_hex("98"),  # T 152 19 << 3
    "sha256_hash": binascii.a2b_hex("b8"),  # X 184 23 << 3
    "muxed_account": binascii.a2b_hex("60"),  # M 96 12 << 3
}

# https://github.com/stellar/js-stellar-base/pull/330/files
# https://github.com/stellar/js-stellar-base/pull/327/files

class StrKey:
    """StrKey is a helper class that allows encoding and decoding strkey.

    """

    @staticmethod
    def encode_ed25519_public_key(data: bytes) -> str:
        """Encodes data to strkey ed25519 public key.

        :param data: data to encode
        :return: strkey ed25519 public key
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return encode_check("ed25519_public_key", data)

    @staticmethod
    def decode_ed25519_public_key(data: str) -> bytes:
        """Decodes strkey ed25519 public key to raw data.

        :param data: strkey ed25519 public key
        :return: raw bytes
        :raises:
            :exc:`Ed25519PublicKeyInvalidError <stellar_sdk.exceptions.Ed25519PublicKeyInvalidError>`
        """
        try:
            return decode_check("ed25519_public_key", data)
        except Exception:
            raise Ed25519PublicKeyInvalidError(
                "Invalid Ed25519 Public Key: {}".format(data)
            )

    @staticmethod
    def is_valid_ed25519_public_key(public_key: str) -> bool:
        """Returns True if the given Stellar public key is a valid ed25519 public key.

        :param public_key: strkey ed25519 public key to check
        :return: `True` if the given Stellar public key is a valid ed25519 public key.
        """
        return is_valid("ed25519_public_key", public_key)

    @staticmethod
    def encode_ed25519_secret_seed(data: bytes) -> str:
        """Encodes data to strkey ed25519 seed.

        :param data: data to encode
        :return: strkey ed25519 seed
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return encode_check("ed25519_secret_seed", data)

    @staticmethod
    def decode_ed25519_secret_seed(data: str) -> bytes:
        """Decodes strkey ed25519 seed to raw data.

        :param data: strkey ed25519 seed
        :return: raw bytes
        :raises:
            :exc:`Ed25519SecretSeedInvalidError <stellar_sdk.exceptions.Ed25519SecretSeedInvalidError>`
        """
        try:
            return decode_check("ed25519_secret_seed", data)
        except Exception:
            raise Ed25519SecretSeedInvalidError(
                "Invalid Ed25519 Secret Seed: {}".format(data)
            )

    @staticmethod
    def is_valid_ed25519_secret_seed(seed: str):
        """Returns True if the given Stellar secret key is a valid ed25519 secret seed.

        :param seed: strkey ed25519 secret seed
        :return: `True` if the given Stellar secret key is a valid ed25519 secret seed.
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return is_valid("ed25519_secret_seed", seed)

    @staticmethod
    def encode_pre_auth_tx(data: bytes) -> str:
        """Encodes data to strkey preAuthTx.

        :param data: data to encode
        :return: strkey preAuthTx
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return encode_check("pre_auth_tx", data)

    @staticmethod
    def decode_pre_auth_tx(data: str) -> bytes:
        """Decodes strkey PreAuthTx to raw data.

        :param data: strkey preAuthTx
        :return: raw bytes
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return decode_check("pre_auth_tx", data)

    @staticmethod
    def encode_sha256_hash(data: bytes) -> str:
        """Encodes data to strkey sha256 hash.

        :param data: data to encode
        :return: strkey sha256 hash
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return encode_check("sha256_hash", data)

    @staticmethod
    def decode_sha256_hash(data: str) -> bytes:
        """Decodes strkey sha256 hash to raw data.

        :param data: strkey sha256 hash
        :return: raw bytes
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        return decode_check("sha256_hash", data)

    @staticmethod
    def encode_muxed_account(data: Xdr.types.MuxedAccount) -> str:
        """Encodes data to strkey muxed account.

        :param data: data to encode
        :return: strkey muxed account
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        if data.type == Xdr.const.KEY_TYPE_ED25519:
            return StrKey.encode_ed25519_public_key(data.ed25519)
        packer = Xdr.StellarXDRPacker()
        packer.pack_int64(data.med25519.id)
        packer.pack_uint256(data.med25519.ed25519)
        data = packer.get_buffer()
        return encode_check("muxed_account", data)

    @staticmethod
    def decode_muxed_account(data: str) -> Xdr.types.MuxedAccount:
        """Decodes strkey muxed account to raw data.

        :param data: strkey muxed account
        :return: raw bytes
        :raises:
            :exc:`ValueError <stellar_sdk.exceptions.ValueError>`
            :exc:`TypeError <stellar_sdk.exceptions.TypeError>`
        """
        data_length = len(data)
        if data_length == 56:
            muxed = Xdr.types.MuxedAccount(
                type=Xdr.const.KEY_TYPE_ED25519,
                ed25519=StrKey.decode_ed25519_public_key(data),
            )
        elif data_length == 69:
            # let's optimize it in v3.
            try:
                xdr = decode_check("muxed_account", data)
            except Exception:
                raise MuxedEd25519AccountInvalidError(
                    "Invalid Muxed Account: {}".format(data)
                )
            unpacker = Xdr.StellarXDRUnpacker(xdr)
            med25519 = Xdr.nullclass()
            med25519.id = unpacker.unpack_int64()
            med25519.ed25519 = unpacker.unpack_uint256()
            muxed = Xdr.types.MuxedAccount(
                type=Xdr.const.KEY_TYPE_MUXED_ED25519, med25519=med25519
            )
        else:
            raise ValueError("Invalid encoded string, this is not a valid account.")
        return muxed

def decode_check(version_byte_name: str, encoded: str) -> bytes:
    encoded_data = _bytes_from_decode_data(encoded)
    encoded_data = encoded_data + b"=" * ((4 - len(encoded_data) % 4) % 4)

    try:
        decoded_data = base64.b32decode(encoded_data)
    except binascii.Error:
        raise ValueError("Incorrect padding.")

    if encoded_data != base64.b32encode(decoded_data):  # Is that even possible?
        raise ValueError("Invalid encoded bytes.")

    version_byte = decoded_data[0:1]
    payload = decoded_data[0:-2]
    data = decoded_data[1:-2]
    checksum = decoded_data[-2:]

    expected_version = _version_bytes.get(version_byte_name)
    if expected_version is None:
        raise TypeError(
            '{} is not a valid version byte name. expected one of "ed25519_public_key", '
            '"ed25519_secret_seed", "pre_auth_tx", "sha256_hash", "muxed_account"'.format(
                version_byte_name
            )
        )

    if version_byte != expected_version:
        raise TypeError(
            "Invalid version byte. Expected {}, got {}".format(
                str(expected_version), str(version_byte)
            )
        )

    expected_checksum = _calculate_checksum(payload)
    if expected_checksum != checksum:
        raise ValueError("Invalid checksum")

    return data

def encode_check(version_byte_name: str, data: bytes) -> str:
    if data is None:
        raise ValueError("cannot encode null data")

    version_byte = _version_bytes.get(version_byte_name)
    if version_byte is None:
        raise TypeError(
            '{} is not a valid version byte name. expected one of "ed25519_public_key", '
            '"ed25519_secret_seed", "pre_auth_tx", "sha256_hash", "muxed_account"'.format(
                version_byte_name
            )
        )
    payload = version_byte + data
    crc = _calculate_checksum(payload)
    return base64.b32encode(payload + crc).decode("utf-8").rstrip("=")

def is_valid(version_byte_name: str, encoded: str) -> bool:
    if encoded and len(encoded) != 56:
        return False
    try:
        decoded = decode_check(version_byte_name, encoded)
        # if len(decoded) != 32:
        #     return False
    except (ValueError, TypeError):
        return False
    return True

def _bytes_from_decode_data(s: Union[str, bytes, bytearray]) -> bytes:
    """copy from base64._bytes_from_decode_data
    """
    bytes_types = (bytes, bytearray)  # Types acceptable as binary data
    if isinstance(s, str):
        try:
            return s.encode("ascii")
        except UnicodeEncodeError:
            raise ValueError("string argument should contain only ASCII characters")
    if isinstance(s, bytes_types):
        return s
    try:
        return memoryview(s).tobytes()
    except TypeError:
        raise TypeError(
            "argument should be a bytes-like object or ASCII "
            "string, not %r" % s.__class__.__name__
        ) from None

def _calculate_checksum(payload):
    # This code calculates CRC16-XModem checksum of payload
    checksum = crc16xmodem(payload)
    # Ensure that the checksum is in LSB order.
    checksum = struct.pack("<H", checksum)
    return checksum


import os
import torchaudio as ta
from Demucs import *
import scipy.io.wavfile
import numpy as np

def create_noise_file(audio_clean,noise_path,output):
    noise, _ = ta.load(str(noise_path), normalize=True)
    audio_length = audio_clean.shape[-1]
    noise_length = noise.shape[-1]
    if noise_length > audio_length:
        noise = noise[..., 0: audio_length]
        clean = np.transpose(audio_clean.detach().cpu().numpy(), (1, 0))
        scipy.io.wavfile.write(output+"_original.wav", samplerate, clean)
    elif noise_length < audio_length:
        audio_clean = audio_clean[..., 0: noise_length]
        clean = np.transpose(audio_clean.detach().cpu().numpy(), (1, 0))
        scipy.io.wavfile.write(output+"_original.wav", samplerate, clean)
        #noise = torch.cat([noise, torch.zeros((noise.shape[0], audio_length - noise_length))], dim=-1)

    return audio_clean + noise_power*noise

def denoise(audio_list,noise_list,output_path):
    for i,audio_path in enumerate(audio_list):
        output = output_path + "/" + str(i)
        wavS, _ = ta.load(str(audio_path), normalize=True)
        if createnoise == 1:
            wavS = create_noise_file(wavS,noise_list[i],output)
            noised = np.transpose(wavS.detach().cpu().numpy(), (1, 0))
            scipy.io.wavfile.write(output+"_noised.wav", samplerate, noised)
        else:
            noised = np.transpose(wavS.detach().cpu().numpy(), (1, 0))
            scipy.io.wavfile.write(output+"_noised.wav", samplerate, noised)

        lenofwave = wavS.shape[1]
        lenofseg = samplerate*segemntlength
        steps = int(lenofwave/lenofseg)
        rest = lenofwave - steps * lenofseg
        listofsplits = []
        for i in range(steps):
            listofsplits.append(wavS[0][i*lenofseg:(i+1)*lenofseg])
        restarr = torch.cat([wavS[0][steps*lenofseg:], torch.zeros(lenofseg - rest)], dim=-1)
        listofsplits.append(restarr)
        stacked = torch.stack(listofsplits)
        denoised = model(stacked.view([-1,1,lenofseg]).to(device))
        denoised = denoised.view([1,1,-1])
        denoised = denoised[0][0][:-int(lenofseg - rest)]
        denoised = np.transpose(denoised.view([1,-1]).detach().cpu().numpy(), (1, 0))
        scipy.io.wavfile.write(output+"_denoised.wav", samplerate, denoised)

if __name__ == "__main__":
    audio_list = ["D:/LibriSpeech/cleandataset/38/4.flac",
                  "D:/LibriSpeech/cleandataset/302/0.flac",
                  "D:/LibriSpeech/cleandataset/369/1.flac",
                  "D:/LibriSpeech/cleandataset/696/1.flac",]
    noise_list = ["D:/musan/musan/noise/free-sound/noise-free-sound-0006.wav",
                  "D:/musan/musan/noise/free-sound/noise-free-sound-0147.wav",
                  "D:/musan/musan/noise/free-sound/noise-free-sound-0155.wav",
                  "D:/musan/musan/noise/free-sound/noise-free-sound-0654.wav",]
    createnoise = 1
    model_path = {"HDemucs_mrstft": "./current_models_Sweeprun_mrstft_HDemucs/G_5_004000.pth",
                  "Demucs_mrstft": "./current_models_Sweeprun_mrstft_Demucs/G_5_004000.pth",
                  "HDemucs_l1": "./current_models_Sweeprun_l1_HDemucs/G_5_004000.pth",
                  "Demucs_l1": "./current_models_Sweeprun_l1_Demucs/G_5_004000.pth",
                  "HDemucs_ranstft": "./current_models_Sweeprun_ranstft_HDemucs/G_5_004000.pth",
                  "Demucs_ranstft": "./current_models_Sweeprun_ranstft_Demucs/G_5_004000.pth"}
    samplerate = 16000
    segemntlength = 2
    noise_power = 0.3
    if not os.path.exists(f'./output'):
        os.mkdir(f'./output')

    for i in model_path:
        output_path = "./output/"+i
        if not os.path.exists(output_path):
            os.mkdir(output_path)
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        if i[0] == "H":
            model = HDemucs().to(device)
        else:
            model = Demucs().to(device)
        model.eval()
        tempmodel = torch.load(model_path[i], map_location=torch.device('cpu'))
        # G.load_state_dict(torch.load(args.G_path, map_location=torch.device('cpu')), strict=True)
        from collections import OrderedDict

        new_state_dict = OrderedDict()
        for k, v in tempmodel.items():
            name = k[7:]  # remove `module.`
            new_state_dict[name] = v
        # load params
        model.load_state_dict(new_state_dict)
        denoise(audio_list,noise_list,output_path)


import mediapipe as mp
import numpy as np
import cv2
import torch
import pyvirtualcam as pv
from os import system
from time import sleep

from tha2.poser.modes import mode_20
from tha2.util import resize_PIL_image, \
    extract_PIL_image_from_filelike, \
    extract_pytorch_image_from_PIL_image, \
    convert_output_image_from_torch_to_numpy

from utils import Point
import calc
import config

class Translator:
    def __init__(self, poser, cuda):
        self.poser = poser
        self.cuda = cuda

        self.face_mesh = mp.solutions.face_mesh.FaceMesh(
            False, 1, True,
            config.min_detection_confidence,
            config.min_tracking_confidence
        )
        self.mp_drawing = mp.solutions.drawing_utils
        self.drawing_spec = self.mp_drawing.DrawingSpec(thickness=1, circle_radius=1)

        pil_img = resize_PIL_image(extract_PIL_image_from_filelike('./data/illust/9e.png'))
        self.source_img = extract_pytorch_image_from_PIL_image(pil_img).to(self.cuda)

        self.cam = cv2.VideoCapture(0)
        self.install_webcam()

    def __del__(self):
        self.cam.release()
        cv2.destroyAllWindows()
        print('tracking terminated')

    # 启动本机摄像头，读取图像
    def run(self):
        while self.cam.isOpened():
            try:
                success, img = self.cam.read()
                if not success:
                    print('Ignoring empty frame...')
                    continue

                landmarks = self.process_img(img)
                if len(landmarks) != 0:
                    img = self.translate_to_anime(landmarks)
                    self.webcam.send(img)
                    self.webcam.sleep_until_next_frame()

            except KeyboardInterrupt:
                break

    # 翻译数据
    def translate_to_anime(self, landmarks):
        if len(landmarks) == 0:
            return

        params = [0.0 for i in range(config.num_params)]
        params[40] = calc.x_angle(landmarks)
        params[39] = calc.y_angle(landmarks, params[40])
        params[41] = calc.z_angle(landmarks)
        params[14] = calc.eye_open(landmarks, True, params[41])
        params[13] = calc.eye_open(landmarks, False, params[41])
        params[26] = calc.mouth_open(landmarks)
        params[38] = calc.iris_ang(landmarks)

        pose = torch.tensor(params, device=self.cuda)
        output_img = self.poser.pose(self.source_img, pose, 0)[0].detach().cpu()
        numpy_img = np.uint8(np.rint(convert_output_image_from_torch_to_numpy(output_img) * 255.0))[:, :, :3]
        # numpy_img = cv2.cvtColor(cv2.flip(numpy_img, 1), cv2.COLOR_BGR2RGB)

        # cv2.imshow('anime', numpy_img)
        # cv2.waitKey(1)

        return numpy_img

    # 解析面部数据点
    def process_img(self, img):
        landmarks = []
        img = cv2.cvtColor(cv2.flip(img, 1), cv2.COLOR_BGR2RGB)
        img.flags.writeable = False
        h, w, c = img.shape
        results = self.face_mesh.process(img)

        if results.multi_face_landmarks:
            total_landmarks = results.multi_face_landmarks[0]
            for idx, landmark in enumerate(total_landmarks.landmark):
                x, y = landmark.x * w, landmark.y * h
                # cv2.circle(img, (int(x), int(y)), 1, (0, 255, 255), -1)
                # cv2.putText(img, str(idx), (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 0.2, (0, 255, 255), 1)
                landmarks.append(Point(x, y))

        # cv2.imshow('face', img)
        # cv2.waitKey(1)

        return landmarks

    # 创建/获取虚拟摄像头
    def install_webcam(self):
        try:
            self.webcam = pv.Camera(width=256, height=256, fps=20, device='VirtualCamera')
        except RuntimeError:
            system('.\\vc\\Install.bat')
            error_n = 0
            while True:
                try:
                    self.webcam = pv.Camera(width=256, height=256, fps=20, device='VirtualCamera')
                    break
                except RuntimeError:
                    error_n += 1
                    if error_n > 5:
                        raise RuntimeError('Fail to install webcam')
                    sleep(1)

if __name__ == '__main__':
    cuda = torch.device('cuda')
    poser = mode_20.create_poser(cuda)

    translator = Translator(poser, cuda)
    translator.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan  1 05:19:32 2021

@author: silasjimmy
"""

# Part C: Finding the right amount to save away

num_of_months = 36
semi_annual_raise = .07
r = 0.04
portion_down_payment = 0.25
total_cost = 1000000
down_payment = total_cost * portion_down_payment

starting_salary = int(input("Enter the starting salary: "))

high = 10000
low = 0
guess = (high + low) / 2
num_of_searches = 0
current_savings = 0

while abs(down_payment - current_savings) >= 100:
    current_savings = 0
    portion_saved = guess / 100
    annual_salary = starting_salary

    for i in range(num_of_months):
        monthly_salary = annual_salary / 12
        monthly_savings = monthly_salary * portion_saved
        investment_returns = current_savings * (r / 12)
        current_savings += monthly_savings + investment_returns
        if num_of_months % 6 == 0:
            annual_salary *= (1 + semi_annual_raise)

    if current_savings > down_payment:
        high = guess
    else:
        low = guess

    guess = (high + low) / 2

    num_of_searches += 1

if portion_saved > 1:
    print("It is not possible to pay the down payment in three years.")
else:
    print("Best savings rate:", portion_saved)
    print("Steps in bisection search:", num_of_searches)


import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import sys
import json
import numpy as np
import multiprocessing

import username_generate

manager = multiprocessing.Manager()
shared_dict = manager.dict()

# {sub_name: [(og_post, list of comments, user_name)]}   How to store the data, add the username
# cut it at thrid level comment
# limit to certain number of comments per post (300)
# 100 post limit per subreddit

def scrape(link):
    print('Scraping:', link)
    subreddit = link.split('/')[4]
    name = link.split('/')[7]
    link2 = link.replace('old','www',1)
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(link2)
    time.sleep(0.5)
    run = True
    title = driver.find_element(By.XPATH, '/html/body/shreddit-app/div/div[2]/shreddit-post/div[2]')
    title = title.text
    driver.get(link)
    time.sleep(0.5)
    driver.execute_script("window.scrollTo(0, 999999999999999999999999999999999999);")
    count = 0
    count2 = 0
    while(run):
        try:
            time.sleep(1)
            button_elements = driver.find_elements(By.CLASS_NAME,"button")
            if(len(button_elements) > 0):
                driver.execute_script("arguments[0].scrollIntoView(true);", button_elements[len(button_elements)-1])
                button_elements[len(button_elements)-1].click()
                count2 += 1
            time.sleep(1)
            button_elements = driver.find_elements(By.CLASS_NAME,"button")

            if(len(button_elements) == 0 or count2 > 10):
                run = False
        except:
            run = False

    comments_list = []
    first_level_comments = driver.find_elements(By.XPATH,"/html/body/div/div/div/div/div/form/div/div")
    # first_level_points = driver.find_elements(By.XPATH,"/html/body/div/div/div/div/div/p/span[3]")
    for i in range(len(first_level_comments)):
        if count >= 300:
            # [subreddit name, (name, title, username of poster, comments list)]
            return [subreddit,(name, title, username_generate.generate_username() ,comments_list)]
        first_comment = first_level_comments[i].find_elements(By.TAG_NAME, "p")
        first_text = ''
        for comment in first_comment:
            first_text += comment.text + ' '
        comments_list.append((username_generate.generate_username(), first_text))
        count += 1
        second_level_xpath = "/html/body/div[4]/div[2]/div[3]/div[" + str(1 + 2*i) + "]/div[3]/div/div[1]/div[2]/form/div/div"
        # second_points_xpath = "/html/body/div[4]/div[2]/div[3]/div[" + str(1 + 2*i) + "]/div[3]/div/div[1]/div[2]/p/span[3]"
        third_level_xpath = "/html/body/div[4]/div[2]/div[3]/div[" + str(1 + 2*i) + "]/div[3]/div/div[1]/div[2]/div/div[1]/div[2]/form/div/div"
        # third_points_xpath = "/html/body/div[4]/div[2]/div[3]/div[" + str(1 + 2*i) + "]/div[3]/div/div[1]/div[2]/div/div[1]/div[2]/p/span[3]"
        second_comments = driver.find_elements(By.XPATH, second_level_xpath)
        if len(second_comments) > 0:
            # second_level_points = driver.find_element(By.XPATH, second_points_xpath)
            second_text = ''
            second_comment = second_comments[0].find_elements(By.TAG_NAME, 'p')
            for comment in second_comment:
                second_text += comment.text + ' '
            comments_list.append((username_generate.generate_username(), second_text))
            count += 1
            third_comments = driver.find_elements(By.XPATH, third_level_xpath)
            if len(third_comments) > 0:
                # third_level_points = driver.find_element(By.XPATH, third_points_xpath)
                third_text = ''
                third_comment = third_comments[0].find_elements(By.TAG_NAME, 'p')
                for comment in third_comment:
                    third_text += comment.text + ' '
                comments_list.append((username_generate.generate_username(), third_text))
                count += 1

    driver.quit()
    return [subreddit,(name, title, username_generate.generate_username() ,comments_list)]

def generateHrefs(num,link):
    if link.startswith('https://www.reddit.com'):
            link = link.replace('://www', '://old')
    print('Generating Hrefs')
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(link)
    time.sleep(0.5)
    count = 0
    hrefs = []
    while(count <= num):
        elements = driver.find_elements(By.CLASS_NAME, "title")
        for element in elements:
            try:
                title = element.find_element(By.TAG_NAME, "a")
                href = title.get_attribute('href')
                if href.startswith('https://old.reddit.com/r/'):
                    hrefs.append(href)
                    count += 1
                elif href.startswith('https://www.reddit.com/r/'):
                    new_href = href.replace('://www', '://old')
                    hrefs.append(new_href)
                    count += 1
                if count >= num:
                    return hrefs
            except:
                pass
        next_button = driver.find_elements(By.CLASS_NAME, "next-button")
        if not len(next_button) == 0:
            button = next_button[0].find_element(By.TAG_NAME, "a")
            button.click()
            time.sleep(0.1)
        else:
            return hrefs
    return hrefs

def sub(hrefs):
    for href in hrefs:
        comments = scrape(href)
        subreddit = comments[0]
        if subreddit in shared_dict:
            shared_dict[subreddit].append(comments[1])
        else:
            shared_dict[subreddit] = [comments[1]]

def main():

    # pass link to desired reddit/subreddit page as first command line argument
    # pass number of desired reddit pages to scrape as second command line argument

    # example command to run
    # /bin/python3 /home/spumel/web-scraper-python/materials-web-scraping-tutorial/scrape_selenium.py https://old.reddit.com/ 100

    subreddit = sys.argv[1]
    num = int(sys.argv[2])
    hrefs = generateHrefs(num,subreddit)
    lists = np.array_split(hrefs,10)
    processes = []
    for list in lists:
        process = multiprocessing.Process(target=sub, args=(list,))
        processes.append(process)

    for process in processes:
        process.start()

    for process in processes:
        process.join()

    result_dict = dict(shared_dict)

    json_data = json.dumps(result_dict)

    with open('out.txt', 'w') as file:
        file.write(json_data)

    print('Data successfully saved to out.txt')

if __name__ == "__main__":
    main()

# Scraped 100 pages in 9 mins


import sys

sys.path.append("src")
import numpy.testing as npt
from geometry.smooth import surfaces
import numpy as np
from geometry.discrete.surfaces import TriangleMesh
import math
from thirdparty.octasphere import octasphere, icosphere
import open3d

sphere = TriangleMesh.from_o3d(open3d.geometry.TriangleMesh.create_sphere())
torus = TriangleMesh.from_o3d(open3d.geometry.TriangleMesh.create_torus())

def test_gauss_bonnet2():
    """
    assert the relationship between gaussian
    curvature and topology
    """
    for mesh in [sphere, torus]:
        E = mesh.euler_characteristic()
        G = mesh.total_gaussian_curvature()

        Ge = E * 2 * math.pi

        np.testing.assert_almost_equal(G, E * 2 * math.pi, 5)

def test_gauss_bonnet1():
    """ Test the theorum that total guassian curvature
    of a closed surface equa ls 2*pi*euler characteristic
    """
    SPHERE_EULER_CHARACTERISTIC = 2
    TORUS_EULER_CHARACTERISTIC = 0

    np.testing.assert_almost_equal(
        sphere.euler_characteristic(),
        SPHERE_EULER_CHARACTERISTIC,
        decimal=5
    )
    np.testing.assert_almost_equal(
        torus.euler_characteristic(),
        TORUS_EULER_CHARACTERISTIC,
        decimal=5
    )

def test_discrete_smooth_total_gaussian_curvature():
    total_smooth   = surfaces.Sphere(2).total_gaussian_curvature(100)
    total_discrete = sphere.total_gaussian_curvature()

    np.testing.assert_almost_equal(total_smooth, total_discrete, decimal=1)

    # smooth_sphere = surfaces.Sphere()
    # discrete_sphere = TriangleMesh.from_surface(smooth_sphere, 100)

    # uv = smooth_sphere.coordinates(100)

    # smooth_curvatures = smooth_sphere.gaussian_curvature(uv)
    # discrete_cuvatures = discrete_sphere.vertex_angle_defects()
    # import pdb; pdb.set_trace()

    # surface = Sphere(2)
    # total = surface.total_gaussian_curvature(500)
    # np.testing.assert_almost_equal(total, 2 * math.pi * SPHERE_EULER_CHARACTERISTIC, decimal=1)

    # surface = EllipsoidLatLon()
    # total = surface.total_gaussian_curvature(500)
    # np.testing.assert_almost_equal(total, 2 * math.pi * SPHERE_EULER_CHARACTERISTIC, decimal=1)

    # surface = Torus()
    # total = surface.total_gaussian_curvature(500)
    # np.testing.assert_almost_equal(total, 2 * math.pi * TORUS_EULER_CHARACTERISTIC, decimal=1)