from torch import nn
from torch.nn import functional as f
import torch
import torchvision

class ConvBnRelu(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_channels, out_channels, 3, padding=1),
                                  nn.BatchNorm2d(out_channels),
                                  nn.ReLU(inplace=True)
                                  )

    def forward(self, x):
        return self.conv(x)

class Decoder(nn.Module):
    def __init__(self, in_channels, middle_channels, out_channels):
        super(Decoder, self).__init__()
        self.deconv = nn.Sequential(
            ConvBnRelu(in_channels, middle_channels),
            nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2, padding=1),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        )

    def forward(self, x):
        return self.deconv(x)

class CSE(nn.Module):
    def __init__(self, in_ch, r):
        super(CSE, self).__init__()

        self.linear_1 = nn.Linear(in_ch, in_ch // r)
        self.linear_2 = nn.Linear(in_ch // r, in_ch)

    def forward(self, x):
        input_x = x

        x = x.view(*(x.shape[:-2]), -1).mean(-1)
        x = f.relu(self.linear_1(x), inplace=True)
        x = self.linear_2(x)
        x = x.unsqueeze(-1).unsqueeze(-1)
        x = torch.sigmoid(x)

        x = input_x * x

        return x

class SSE(nn.Module):
    def __init__(self, in_ch):
        super(SSE, self).__init__()

        self.conv = nn.Conv2d(in_ch, 1, kernel_size=1, stride=1)

    def forward(self, x):
        input_x = x

        x = self.conv(x)
        x = torch.sigmoid(x)

        x = input_x * x

        return x

class SCSE(nn.Module):
    def __init__(self, in_ch, r):
        super(SCSE, self).__init__()

        self.cSE = CSE(in_ch, r)
        self.sSE = SSE(in_ch)

    def forward(self, x):
        c_se = self.cSE(x)
        s_se = self.sSE(x)

        x = c_se + s_se

        return x

class UNetResNet(nn.Module):
    def __init__(self, out_channel, dropout=0.2,
                 pretrained=False):
        super().__init__()
        self.num_classes = out_channel
        self.dropout = dropout

        self.encoder = torchvision.models.resnet34(pretrained=pretrained)
        self.pool = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)

        self.se_e0 = SCSE(64, 4)
        self.conv1 = self.encoder.layer1
        self.se_e1 = SCSE(64, 4)
        self.conv2 = self.encoder.layer2
        self.se_e2 = SCSE(128, 4)
        self.conv3 = self.encoder.layer3
        self.se_e3 = SCSE(256, 4)
        self.conv4 = self.encoder.layer4
        self.se_e4 = SCSE(512, 4)

        self.max_pool = nn.MaxPool2d(2, 2)

        self.center = nn.Sequential(ConvBnRelu(512, 1024))

        self.dec5 = nn.ConvTranspose2d(1024, 512, kernel_size=4, stride=2, padding=1)

        self.dec4 = Decoder(512, 512, 256)
        self.se_d4 = SCSE(256, 4)
        self.dec3 = Decoder(256 + 256, 512, 128)
        self.se_d3 = SCSE(256, 4)
        self.dec2 = Decoder(128 + 128, 256, 64)
        self.se_d2 = SCSE(64, 4)
        self.dec1 = Decoder(64 + 64, 128, 64)
        self.se_d1 = SCSE(128, 4)
        self.final = nn.Conv2d(64, out_channel, kernel_size=1)

    def forward(self, x):
        x = self.encoder.conv1(x)
        x = self.encoder.bn1(x)
        x = self.encoder.relu(x)
        x = self.pool(x)
        x = self.se_e0(x)
        conv1 = self.conv1(x)
        conv1 = self.se_e1(conv1)
        conv2 = self.conv2(conv1)
        conv2 = self.se_e2(conv2)
        conv3 = self.conv3(conv2)
        conv3 = self.se_e3(conv3)
        conv4 = self.conv4(conv3)
        conv4 = self.se_e4(conv4)

        pool = self.max_pool(conv4)
        center = self.center(pool)
        dec5 = self.dec5(center)

        dec4 = self.dec4(torch.cat([dec5, conv4], 1))
        dec4 = self.se_d4(dec4)
        dec3 = self.dec3(torch.cat([dec4, conv3], 1))
        dec3 = self.se_d3(dec3)
        dec2 = self.dec2(torch.cat([dec3, conv2], 1))
        dec2 = self.se_d2(dec2)
        dec1 = f.dropout2d(self.dec1(torch.cat([dec2, conv1], 1)), p=self.dropout)
        dec1 = self.se_d1(dec1)
        return self.final(dec1)


#Import cssutils and webcolors as required
import cssutils as cu
import webcolors as wc

#Found this online, it suppresses the ridiculous level of error logging from cssutils
import logging
cu.log.setLevel(logging.CRITICAL)

#Giving a shorter name to the name-to-hex dictionary
nth = wc.CSS2_NAMES_TO_HEX

#I'll keep the next few lines in for now, shows how I figured out how to only
#convert a color value to hex if there's a hex representation for that value
# blue = 'blue'
# blurb = 'blurb'
# if blue in nth:
#     print(nth[blue])
# if blurb in nth:
#     print(nth[blurb])

#Read from the css file provided
inbound_file = cu.parseFile('main.css')

#For each rule in our file...
for rule in inbound_file.cssRules:
    #If it's a style rule...
    if rule.type == rule.STYLE_RULE:
        #Then check out the properties it has
        for property in rule.style:
            #And if it's either color or background-color and has a name-to-hex value
            if (property.name == 'color' or property.name == 'background-color') and property.value in nth:
                #Then change it's value to the hex value
                property.value = nth[property.value]

#Get a new file ready to write to
outbound_file = open("new_main.css","w")
#Decode the bytestring values of the outbound file and encode as UTF-8
outbound_file.write(inbound_file.cssText.decode("utf-8"))
#Finally, close the file
outbound_file.close()


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myCv.settings')

import django
django.setup()

from faker import Faker
fake = Faker()

from accounts.models import User
from groups.models import Group, GroupMember
from posts.models import Post
import random

uName = []
gName = []

def initializeList():
    for i in range(10):
        uName.append(fake.profile(fields=['username'])['username'])

    for i in range(10):
        gName.append(fake.sentence(nb_words=6))

def addUser():
    user = User.objects.get_or_create(username=random.choice(uName))[0]
    user.save()
    user.set_password(fake.word())
    user.save()
    return user

def addGroup():
    name = random.choice(gName)
    admin = addUser()
    desc = fake.paragraph(nb_sentences=3)
    group = Group.objects.get_or_create(name=name)[0]
    group.admin=admin
    group.desc=desc
    group.save()
    return group

def joinGroup():
    user = addUser()
    group = addGroup()
    gMember = GroupMember.objects.get_or_create(member=user, group=group)[0]
    gMember.save()

def addPost():
    message = fake.text(max_nb_chars=200, ext_word_list=None)
    publisher = addUser()
    group = addGroup()
    post = Post(publisher=publisher, group=group, message=message)
    post.save()

def populate(n):
    initializeList()
    for i in range(n):
        joinGroup()
        addPost()

if(__name__=='__main__'):
    print('\n Populating Fake DAta !!!')
    populate(15)
    print('\n Fake data is inserted...\n')


import sklearn.metrics
import numpy as np
import matplotlib.pyplot as plt
# MCC-F1 Funtion:
def mcc_f1(ground_truths, predicted_values):

    """
    The function mcc_f1 calculates MCC and F1 scores based on varying thresholds identifed by the PRC function
    based on inputs -  ground truth vaules and predicted values.

    Parameters
    ----------
    ground_truths, predicted_values : numpy.ndarray;
                        Input arrays; groundtruth values (True values) and predicted values.

    Returns
    -------
    mcc, f1, thresholds: numpy.ndarray;
                           Output arrays; Unit normalized MCC scores and F1 score values
                           for every threshold.
    """

    mcc = []
    f1 = []

    # Generating thresholds with the PRC function
    precision, recall, thresholds = sklearn.metrics.precision_recall_curve(
                                    ground_truths, predicted_values)

    # For every value of the thresholds (cutoffs), calculate MCC and F1 scores :
    for T in thresholds:
        predicted_values_after_cutoff = predicted_values > T  # Positive classes are determined for every predicted value that exceeds the threshold.

        mcc.append((sklearn.metrics.matthews_corrcoef(
                    ground_truths, predicted_values_after_cutoff)+1)*0.5) # get normalised MCC: change the range of MCC from [-1, 1] to [0, 1]

        f1.append(sklearn.metrics.f1_score(
                    ground_truths, predicted_values_after_cutoff))

    mcc = np.array(mcc)
    f1 = np.array(f1)

    # Resulting values are an array of MCC & F1 scores across different thresholds (cutoffs)
    return mcc,f1, thresholds

# MCC-F1 Metric:
def MCC_F1_Metric(mcc,f1,thresholds, W_num_of_subranges = 100):

    """
    MCC_F1_Metric function computes the MCC-F1 metric i.e. the average distances between the predcition scores to the point of Perfect Performace (1,1)
    and also results in the best threshold value

    Parameters
    ----------

    mcc,f1, thresholds : numpy.ndarray;
                        MCC and F1 scores across varying thresholds

    W_num_of_subranges : int, optional;
        Number of subranges- default = 100;
        larger values of 𝑊 will cause the MCC-𝐹1 metric to capture the performance of a classifier more accurately.

    Returns
    -------
    MCC_F1_Met : float;
                The ratio of the average distance of the MCC-F1 score to the Point of Perfect Performace (1,1)
                A metric to compare classifier performance.

    Best_threshold : int;
                Index of the best prediction score threshold (𝑇),
                Index of the point on the MCC-𝐹1 curve closest to the point of perfect performance (1,1).

    """

    subrange_intervals = np.linspace(np.min(mcc), np.max(mcc),W_num_of_subranges) # breaking into subranges

    # Computing MCC values per subrange

    values_per_subrange = np.zeros_like(subrange_intervals) # 'n' is the number of points per subrange
    for i in range(W_num_of_subranges-1):
        for j in mcc:
            if j >= subrange_intervals[i] and j < subrange_intervals[i+1]:
                values_per_subrange[i] = values_per_subrange[i] + 1

    # Calculating the distances between points in a subrange to the Point of perfect performance (1,1)
    Distance_of_points_within_subrange = np.sqrt(((mcc-1)**2)+((f1-1)**2)) # calculating the Euclidean distance 𝐷𝑖
    sum_of_distance_within_subrange = np.zeros_like(subrange_intervals)
    index = -1
    for value in mcc:
        index += 1
        for i in range(W_num_of_subranges-1):
                 if value >= subrange_intervals[i] and value < subrange_intervals[i+1]:
                        sum_of_distance_within_subrange[i] = sum_of_distance_within_subrange[i] + Distance_of_points_within_subrange[index]

    # Mean Distance across subranges
    mean_Distance_per_subrange = np.array(sum_of_distance_within_subrange/values_per_subrange)
    total_number_of_subranges = 0
    total_sum_of_mean_distances = 0
    for i in mean_Distance_per_subrange:
        if not np.isnan(i):
            total_number_of_subranges += 1
            total_sum_of_mean_distances += i # addition of all the means across subranges that have atleast 1 MCC value.

    grand_mean_distance = total_sum_of_mean_distances/total_number_of_subranges # P = total number of subranges that have atleast 1 MCC value

    """
    Compare the grand average distance to √2 (The distance between the point of worst performance (0,0) and
    the point of perfect performance (1,1) is √2).That is the maximum possible distance between a point on the MCC-𝐹1 curve
    The ratio between the grand avgerage distance and √2 is taken.
    This ratio ranges between 0 and 1 (worst value = 0; best value = 1). To get the MCC-𝐹1 score, we subtract this ratio from 1
    """

    MCC_F1_Met = 1 - (grand_mean_distance /np.sqrt(2))

    # Finding the best threshold 𝑇 the point on the MCC-𝐹1 curve closest to the point of perfect performance (1,1).

    result = np.where(Distance_of_points_within_subrange == Distance_of_points_within_subrange.min())
    Best_threshold = result[0][0]

    print("MCC-F1 metric = {0:.2f}".format(MCC_F1_Met), "Best Threshold = {0:.2f}".format(thresholds[Best_threshold]))

    return MCC_F1_Met,Best_threshold

# Generating a plot (the overall canvas);
axis = None
def axis_plotting(total_plots):
    global axis
    if axis == None:
        fig = plt.figure(figsize=(7,7))
        axis = []
        for i in range(1,total_plots+1):
            ax = fig.add_subplot(total_plots, 1, i)
            axis.append(ax)

def plotting(mcc, f1, thresholds,MCC_F1_Met,Best_threshold,classifer_name = "",total_plots = 1,ax = 0):

        """
        Plotting function - fuction to plot the MCC and F1 scores (across varying Thresholds), and highlight the the MCC_F1 metric with the best threshold
        For multiple plots, 'total_plots' should be > 1 AND 'ax' (< total_plots) starting from  1

        Parameters
        ----------
        mcc,f1 : numpy.ndarray;
            MCC and F1 scores across varying thresholds

        MCC_F1_Met : float;
                The ratio of the average distance of the MCC-F1 score to the Point of Perfect Performace (1,1)
                A metric to compare classifier performance.

        Best_threshold : int;
                Index of the best prediction score threshold (𝑇),
                Index of the point on the MCC-𝐹1 curve closest to the point of perfect performance (1,1).

        classifer_name : str, optional
                    default = ""

        total_plots: int, optional
                    Number of sub-plots to generate; default = 1

        ax : int, optional
            Index of subplot ([1,2,3...]); default = None

        Returns
        -------

        MCC-F1 curve:
            for multiple sublplots, adjust total_plots and ax input value.

        """

        global axis
        if total_plots > 1 and ax < total_plots: # if more than 1 total plot is mentioned;
            axis_plotting(total_plots) #calling function to generate plot;
            if ax == 0:
                subplot = plt.gca() # plot on current axis value,
            else:
                subplot = axis[ax-1] # plotting additional graphs based on index of the axis.
        elif ax > total_plots:
            print("Index inconsistent with total_plots")
        else:
            axis = None
            subplot = plt.gca() # plot on base plot if no additional subplots are needed

        # Labelling the plot;
        subplot.plot(f1,mcc)
        subplot.annotate(classifer_name + "_MCC_Met - {0:.2f}".format(
            MCC_F1_Met),
                 (f1[Best_threshold],mcc[Best_threshold]),
                 textcoords="offset points",
                 xytext=(0,10),
                 ha='left')

        subplot.scatter(f1[Best_threshold],mcc[Best_threshold],color='red')
        subplot.set(xlabel='F1 score',
               ylabel='unit−normalized MCC',title='MCC-F1 curve')

        # Inserting Random line
        subplot.axhline(0.5,linewidth=0.7, color='green',linestyle='--')
        subplot.annotate('Random line',xy =(0,0.5),xytext=(250,-15),
                    color='g', xycoords = subplot.get_yaxis_transform(),
                    textcoords="offset points", va="bottom")

        # Displaying and labelling points of performances.

        subplot.scatter(1,1,color = 'black');subplot.scatter(0,0,color = 'black')
        subplot.annotate('Point of Perfect performance',(1,1),
                    textcoords="offset points",xytext=(-9,-2),ha='right')
        subplot.annotate('Point of Worst performance',(0,0)
                    ,textcoords="offset points",xytext=(9,2),ha='left')

        subplot.set_xticks(np.arange(0, 1.1, 0.25)); subplot.set_yticks(np.arange(0, 1.1, 0.25))
        subplot.set_xlim(-0.05,1.05); subplot.set_ylim(-0.05,1.05)
        subplot.grid(alpha = 0.3)


# -----------------------------------------------------------------------------
# Project           :   Texto
# Author            :   Sebastien Pierre
# License           :   BSD License (revised)
# -----------------------------------------------------------------------------
# Creation date     :   10-Feb-2006
# Last mod.         :   10-Feb-2006

import os, sys, re

TEST_FILE  = re.compile("(syntax)\-(.+)\.kwi")
TEST_FILES = {}

DIR  = os.path.dirname(os.path.abspath(__file__) ) + "/results"
KIWI = os.path.abspath(DIR + "/../../Sources/texto/main.py")
if not os.path.exists(DIR):
    os.makedirs(DIR)
def do():
	# We populate the test files hash table
	this_dir = os.path.abspath(os.path.dirname(__file__))
	for path in os.listdir(this_dir):
		m = TEST_FILE.match(path)
		if not m: continue
		f = TEST_FILES.setdefault(m.group(1), [])
		f.append((m.group(2), os.path.join(this_dir, path)))

	index_f = file(DIR + "/index.html", "w")
	index_f.write("
Texto test suite

")

	# And now execute the tests
	groups = list(TEST_FILES.keys()) ; groups.sort()
	files  = []
	for group in groups:
		index_f.write("
%s
" % (group))
		for test, test_path in TEST_FILES[group]:
			print("%4s:%25s " % (group, test), end=' ')
			inp, out, err = os.popen3("python " + KIWI + " -m " + test_path)
			dest_path = os.path.splitext(test_path)[0] + ".html"
			f = open(dest_path, "w")
			f.write(out.read())
			f.close()
			index_f.write("
 	%s	[KIWI ⇒ HTML]
 " % (
			test, test_path, dest_path))
			files.append(dest_path)
			err =  err.read()
			if err.strip() == "": print("[OK]")
			else: print("[FAILED]", err)

	index_f.write("
")
	index_f.close()

if __name__ == "__main__":
	if len(sys.argv) == 1:
		do()
	else:
		os.system("rm %s/*.html" % (os.path.dirname(__file__) or "."))
# EOF


import argparse
import os
import torch
from torch import nn, optim
from torch.utils.data import DataLoader
from torchvision.datasets import SVHN
from torchvision import transforms
from torchvision import datasets
from models import CNN
from trainer import train_source_cnn
from utils import get_logger

def main(args):
    args.logdir = args.logdir+args.mode
    if not os.path.exists(args.logdir):
        os.makedirs(args.logdir)
    logger = get_logger(os.path.join(args.logdir, 'train_source.log'))
    logger.info(args)

    # data
    # source_transform = transforms.Compose([
    #     transforms.ToTensor()]
    # )
    # source_dataset_train = SVHN(
    #     './input', 'train', transform=source_transform, download=True)
    # source_dataset_test = SVHN(
    #     './input', 'test', transform=source_transform, download=True)
    # source_train_loader = DataLoader(
    #     source_dataset_train, args.batch_size, shuffle=True,
    #     drop_last=True,
    #     num_workers=args.n_workers)
    # source_test_loader = DataLoader(
    #     source_dataset_test, args.batch_size, shuffle=False,
    #     num_workers=args.n_workers)
    source_dataset_name = 'MNIST'
    target_dataset_name = 'mnist_m'
    source_image_root = os.path.join('dataset', source_dataset_name)
    target_image_root = os.path.join('dataset', target_dataset_name)
    batch_size = 128
    image_size = 28
    img_transform_source = transforms.Compose([
        transforms.Resize(image_size),
        transforms.ToTensor(),
        transforms.Normalize(mean=(0.1307,), std=(0.3081,))
    ])
    dataset_source_train = datasets.MNIST(
        root='dataset',
        train=True,
        transform=img_transform_source,
        download=True
    )
    source_train_loader = torch.utils.data.DataLoader(
        dataset=dataset_source_train,
        batch_size=batch_size,
        shuffle=True,
        num_workers=8)
    dataset_source_test = datasets.MNIST(
        root='dataset',
        train=False,
        transform=img_transform_source,
    )

    source_test_loader = torch.utils.data.DataLoader(
        dataset=dataset_source_test,
        batch_size=batch_size,
        shuffle=False,
        num_workers=8
    )

    # train source CNN
    source_cnn = CNN(in_channels=args.in_channels).to(args.device)
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(
        source_cnn.parameters(),
        lr=args.lr, weight_decay=args.weight_decay)
    source_cnn = train_source_cnn(
        source_cnn, source_train_loader, source_test_loader,
        criterion, optimizer, args=args)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    # NN
    parser.add_argument('--in_channels', type=int, default=3)
    parser.add_argument('--n_classes', type=int, default=10)
    parser.add_argument('--trained', type=str, default='')
    parser.add_argument('--slope', type=float, default=0.2)
    # train
    parser.add_argument('--lr', type=float, default=1e-3)
    parser.add_argument('--weight_decay', type=float, default=2.5e-5)
    parser.add_argument('--epochs', type=int, default=50)
    parser.add_argument('--batch_size', type=int, default=128)
    # misc
    parser.add_argument('--device', type=str, default='cuda:0')
    parser.add_argument('--n_workers', type=int, default=0)
    parser.add_argument('--logdir', type=str, default='outputs/garbage_')
    parser.add_argument('--message', '-m',  type=str, default='')
    parser.add_argument('--mode', type=str, default='s2u',choices=['m2mm','s2u'])
    args, unknown = parser.parse_known_args()
    main(args)


# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/topics/items.html

from scrapy.item import Item, Field

class iTunesItem(Item):
	genreId = Field()
	genreUrl = Field()
	genreName = Field()
	subgenreId = Field()
	subgenreUrl = Field()
	subgenreName = Field()

class PodcastItem(Item):
	uri = Field()
	id = Field()
	audioType = Field()
	indexedDate = Field()
	mediaType = Field()
	images = Field(default=['dummyUrl'])

	itunesArtistId = Field()
	itunesArtistName = Field()
	itunesArtistViewUrl = Field()
	itunesArtworkUrl100 = Field()
	itunesArtworkUrl30 = Field()
	itunesArtworkUrl55 = Field()
	itunesArtworkUrl60 = Field()
	itunesArtworkUrl170 = Field()
	itunesArtworkUrl600 = Field()
	itunesCollectionCensoredName = Field()
	itunesCollectionExplicitness = Field()
	itunesCollectionId = Field()
	itunesCollectionName = Field()
	itunesCollectionPrice = Field()
	itunesCollectionViewUrl = Field()
	itunesCurrency = Field()
	itunesGenreIds = Field()
	itunesGenres = Field()
	itunesHtmlChannel = Field()
	itunesHtmlOwner = Field()
	itunesHtmlDescription = Field()
	itunesPopular = Field()
	itunesPopularInGenre = Field()
	itunesPrimaryGenreName = Field()
	itunesRelated = Field()
	itunesReleaseDate = Field()
	itunesSimilar = Field()
	itunesTrackCensoredName = Field()
	itunesTrackCount = Field()
	itunesTrackExplicitness = Field()
	itunesTrackId = Field()
	itunesTrackName = Field()
	itunesTrackPrice = Field()
	itunesTrackViewUrl = Field()

	rssChannelTitle = Field()
	rssChannelLink = Field()
	rssChannelDescription = Field()
	rssChannelCopyright = Field()
	rssChannelLanguage = Field()
	rssChannelGenerator = Field()
	rssChannelPubDate = Field()
	rssChannelitunesSummary = Field()
	rssChannelitunesSubtitle = Field()
	rssChannelitunesKeywords = Field()
	rssChannelitunesAuthor = Field()
	rssChannelitunesOwnerEmail = Field()
	rssChannelitunesOwnerName = Field()
	rssChannelitunesCategory = Field()
	rssChannelitunesImage = Field()
	rssChannelitunesExplicit = Field()
	rssChannelImageUrl = Field()
	rssChannelImageTitle = Field()
	rssChannelImageLink = Field()
	rssChannelLastBuildDate = Field()

	rssItemTitle = Field()
	rssItemDescription = Field()
	rssItemPubDate = Field(default=['1900-01-01T00:00:00Z'])
	rssItemLink = Field()
	rssItemGuid = Field()
	rssItemAuthor = Field()
	rssItemitunesSubtitle = Field()
	rssItemitunesSummary = Field()
	rssItemitunesKeywords = Field()
	rssItemitunesDuration = Field()
	rssItemitunesExplicit = Field()
	rssItemEnclosureUrl = Field()
	rssItemEnclosureLength = Field()
	rssItemEnclosureType = Field()
	rssItemMediaUrl = Field()
	rssItemMediaFileSize = Field()
	rssItemMediaType = Field()

	episodeId = Field()
	episodeTitle = Field()
	episodeTitleN = Field()
	episodeSubtitle = Field()
	episodeDescription = Field()
	episodeStart = Field()
	episodeEnd = Field()
	episodeDuration = Field()
	episodePublishDate = Field()
	episodePublishYear = Field()
	# Solr has episodePublishYear which is derived from above in DIH
	episodeMimeType = Field()
	episodeMedia = Field()
	episodeImage = Field()
	episodeHomepage = Field()
	episodeFirstBroadcast = Field()
	episodeAvailableUntil = Field()
	episodeTags = Field()
	episodeRelatedTitle = Field()
	episodeRelatedUrl = Field()

	seriesId = Field()
	seriesName = Field()
	seriesNameN = Field()
	seriesDescription = Field()
	seriesImage = Field()
	seriesFeed = Field()
	seriesCurrentItem = Field()
	seriesHomepage = Field()
	seriesTags = Field()

	brandAvgDuration = Field()
	brandCategory = Field()
	brandCurrentItem = Field()
	brandDescription = Field()
	brandExplicit = Field()
	brandFeed = Field() # The on demand feed for the content
	brandFrequency = Field()
	brandGenreIds = Field()
	brandGenres = Field(default='unknown')
	brandHomepage = Field()
	brandId = Field()
	brandIds = Field()
	brandImage = Field()
	brandLanguage = Field()
	brandCulture = Field()
	brandLanguage = Field()
	brandName = Field()
	brandNameN = Field()
	brandPublishDate = Field()
	brandRelated = Field()
	brandRegions = Field()
	brandShortName = Field()
	brandSimilar = Field()
	brandTags = Field()
	brandTimes = Field()

	channelId = Field()
	channelName = Field()
	channelNameN = Field()
	channelTitle = Field()
	channelSubTitle = Field()
	channelDescription = Field()
	channelImage = Field()
	channelFeed = Field() # The live stream for a channel
	channelHomepage = Field()
	channelLocation = Field()
	channelAddress = Field()
	channelTuneInId = Field()
	channelCategory = Field()
	channelGenres = Field()
	channelGenreIds = Field()
	channelPlaylist = Field()
	channelPlaylistHeaders = Field()
	channelFormats = Field()
	channelRelatedGenres = Field()
	channelRelatedGenreIds = Field()
	channelRecommended = Field()
	channelRecommendedDescription = Field()
	channelRecommendedId = Field()
	channelRecommendedFormats = Field()
	channelRecommendedType = Field()
	channelRecommendedImage = Field()
	channelCallSign = Field()
	channelSlogan = Field()
	channelUrl = Field()
	channelLocation = Field()
	channelEmail = Field()
	channelPhone = Field()
	channelLanguage = Field()

	channelTuneInReportUrl = Field()
	channelTuneInDetailUrl = Field()
	channelTuneInIsPreset = Field()
	channelTuneInIsAvailable = Field()
	channelTuneInIsMusic = Field()
	channelTuneInHasSong = Field()
	channelTuneInHasSchedule = Field()
	channelTuneInHasTopics = Field()
	channelTuneInTwitterId = Field()
	channelTuneInItem = Field()
	channelTuneInNowPlayingId = Field()
	channelTuneInPresetId = Field()
	channelTuneInType = Field()
	channelTuneInBitrate = Field()
	channelTuneInReliability = Field()
	channelTuneInGuideId = Field()
	channelTuneInShowId = Field()
	channelTuneInCurrentTrack = Field()
	channelTuneInUrl = Field()
	channelTuneInId = Field()
	channelTuneInDesc = Field()
	channelTuneInLogo = Field()
	channelTuneInAddress = Field()
	channelTuneInEmail = Field()
	channelTuneInLocation = Field()
	channelTuneInName = Field()
	channelTuneInHomepage = Field()
	channelTuneInStreamIds = Field()
	channelTuneInGenres = Field()
	channelTuneInLang = Field()
	channelTuneInContact = Field()
	channelTuneInNetworks = Field()
	channelTuneInNetworkIds = Field()

	#live streams
	#all stream* should be multivalue in Solr
	streamId = Field()
	streamName = Field()
	streamReliability = Field()
	streamUrl = Field()
	streamMediaType = Field()
	streamMedia = Field()
	streamBandwidth = Field()
	streamData = Field() #anything else that dont know where to put but may be useful
	streamRecentlyPlayed = Field()
	streamHistory = Field()
	streamServerName = Field()
	streamGenre = Field()
	streamContentType = Field()
	streamTitle = Field()
	streamHomepage = Field()
	streamStatus = Field()
	streamStatusUrl = Field()

	#audio books
	audiobookProjectName = Field()
	audiobookLibrivoxURL = Field()
	audiobookRssURL = Field()
	audiobookCategory = Field()
	audiobookGenre = Field()
	audiobookAuthor1 = Field()
	audiobookAuthor2 = Field()
	audiobookAuthor3 = Field()
	audiobookAuthor4 = Field()
	audiobookTranslator = Field()
	audiobookLanguage = Field()
	audiobookType = Field()

	ownerId = Field()
	ownerName = Field()
	ownerNameN = Field()
	ownerEmail = Field()
	ownerKey = Field()
	ownerImage = Field()
	ownerHomepage = Field()
	ownerDescription = Field()

	#some meta data fields created locally in spider
	#persisted here to be used later in pipelines
	metaBrandName = Field()
	metaBrandDescription = Field()
	metaGenreName = Field()
	metaChannelName = Field()
	metaOwnerName = Field()


# -*- coding: utf-8 -*-
from django.http import HttpResponse
from django.utils import simplejson as json
from django.conf import settings
import csv, os
from backend.models import Product, Vendor, ItemProduct
from decimal import *
from threading import Thread
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from datetime import datetime

def index(request):

    vendor_id = int(request.POST['vendor'])

    #start upload file async
    Thread(target=handle_uploaded_file, kwargs={"file": request.FILES['product'], "vendor_id": vendor_id, "request": request}).start()

    return HttpResponse(json.dumps({ "success": True, "file": request.FILES['product'].name }), mimetype='text/html')

def handle_uploaded_file(**kwargs):

    file      = kwargs['file']
    vendor_id = kwargs['vendor_id']
    request   = kwargs['request']
    user      = request.user.id

    path = "%s/%s" % (settings.UPLOAD_PRODUCT_DIR, file.name,)

    destination = open(path, 'wb+')

    for chunk in file.chunks():
        destination.write(chunk)
    destination.close()

    csvfile = csv.reader(open(path, 'rU'), delimiter=settings.CSV_DELIMITER)

    #header exist -> skip header
    csvfile.next()

    for row in csvfile:
        price = None

        try:
            product = Product.objects.get(product_num=row[0], vendor__id=vendor_id)
        except ObjectDoesNotExist:
            #there is no such product
            #cleaning
            row = clean(row)

            if len(row) >= 8 and row[7]:
                price = Decimal(row[7].replace(',','.').replace('$','').replace(' ', ''))
            else:
                price = None

            if len(row) >= 9:
                notes = row[8]
            else:
                notes = None

            Product.objects.create(product_num=row[0], upc=row[1], category=row[2], \
                                    name1=row[3], name2=row[4], package=row[5], brand=row[6], \
                                    price=price, notes=notes, date_modified=datetime.now(), vendor_id=vendor_id, user_id=user)
        else:
            #product already exist -> update price and date_modified
            if len(row) >= 8 and row[7]:
                price = Decimal(row[7].replace(',', '.').replace('$', '').replace(' ', ''))
            else:
                price = None

            product.price = price
            product.date_modified = datetime.now()

            product.save()

        updateItemProduct(row[0], vendor_id, price)

    #reindex sphinx, takes many resources, let's not do it here, sphinx reindexes itself daily
    #Thread(target=reindex).start()

    #rename filename
    rename(path, vendor_id, request)

def updateItemProduct(product_num, vendor_id, price):
    try:
        product = ItemProduct.objects.get(product_num=product_num, vendor__id=vendor_id)
    except ObjectDoesNotExist:
        pass
    except MultipleObjectsReturned:
        product = ItemProduct.objects.filter(product_num=product_num, vendor__id=vendor_id).order_by('-date_modified')[0]
    else:
        if price is not None:
            product.price = price
            product.date_modified = datetime.now()

            product.save()

def rename(old_filename, vendor_id, request):

    try:
        vendor = Vendor.objects.get(pk=vendor_id)
    except ObjectDoesNotExist:
        vendor_name = 'unknown_vendor'
    else:
        vendor_name = vendor.name

    #renaming file
    new_filename = "%s/%s__%s__%s.csv" % (settings.UPLOAD_PRODUCT_DIR, request.user, vendor_name, datetime.now().strftime("%Y-%m-%d %H_%M_%S"),)

    os.rename(old_filename, new_filename)

def clean(row):
    clean_row = []

    for col in row:
        if col == '':
            col = None
        else:
            col = cut_none_unicode(col)

        clean_row.append(col)

    return clean_row

def cut_none_unicode(col):
    clean_col = []

    for letter in col:
        try:
            letter = unicode(letter)
        except UnicodeDecodeError:
            pass
        else:
            clean_col.append(letter)

    return ''.join(clean_col)

def reindex():
    import subprocess

    command = "/bin/indexer --config %s/sphinx.conf --rotate --all" % (settings.PROJECT_ROOT,)
    subprocess.call(command, shell = True)


the_list = [0, 2, 1, 9, 8, 5, 6, 7, 2, 310, 20, 12, 34, 100, 99, 30]
print(the_list)

def straightforward_way(in_list):
    largest = in_list[0]
    for item in in_list:
        if item > largest:
            largest = item
    print(f'THe largest item is {largest}')

def shother_code_way(in_list):
    in_list.sort()
    print(f'THe largest item is {in_list[-1]}')

straightforward_way(the_list)

shother_code_way(the_list)


from __future__ import annotations
import collections
import random
import heapq
import math

# """
# 401 / 401 test cases passed.
# Status: Accepted
# Runtime: 62 ms
# You are here!
# Your runtime beats 35.12% of python submissions.
# """
# class Solution(object):
#     def isHappy(self, n):
#         """
#         :type n: int
#         :rtype: bool
#         """
#         n = abs(n)
#         lst = []
#         while n not in lst:
#             lst.append(n)
#             re = 0
#             while n > 0:
#                 re += (n % 10) ** 2
#                 n //= 10
#             n = re
#         return n == 1

# print(Solution().isHappy(49))
# print(Solution().isHappy(-1))

"""
Success
Details
Runtime: 44 ms, faster than 47.32% of Python3 online submissions for Happy Number.
Memory Usage: 12.8 MB, less than 100.00% of Python3 online submissions for Happy Number.
"""
# set
# class Solution:
#     def isHappy(self, n: int) -> bool:
#         seen = set()
#         while n not in seen:
#             seen.add(n)

#             temp = 0
#             while n != 0:
#                 temp += (n % 10) ** 2
#                 n //= 10
#             n = temp

#         return n == 1

"""
Success
Details
Runtime: 32 ms, faster than 95.22% of Python3 online submissions for Happy Number.
Memory Usage: 12.7 MB, less than 100.00% of Python3 online submissions for Happy Number.
"""
class Solution:
    def isHappy(self, n: int) -> bool:
        # brute force O(n) space
        visited = set()
        n = abs(n)
        while n not in visited:
            visited.add(n)
            new_n = 0
            while n != 0:
                new_n += (n % 10) ** 2
                n //= 10
            n = new_n

        return n == 1


# As usual, most of the code is stolen from Jimmy Song's book, I just made it worse

from io import BytesIO

from base58 import *

from op import *

from ecc import *

class Script:

    # The basic constructor; either empty list of commands, or some commands
    def __init__(self, cmds=None):
        if cmds is None:
            self.cmds = []
        else:
            self.cmds = cmds

    # This is just for printing the script for debugging purposes
    def __repr__(self):
        result = []
        for cmd in self.cmds:
            if type(cmd) == int:
                if OP_CODE_NAMES.get(cmd):
                    name = OP_CODE_NAMES.get(cmd)
                else:
                    name = 'OP_[{}]'.format(cmd)
                result.append(name)
            else:
                result.append(cmd.hex())
        return ' '.join(result)

    # We'll want to concatenate two scripts (scriptSig + scriptPubKey)
    # For this we overload the sum operator (+)
    def __add__(self, other):
        return Script(self.cmds + other.cmds)

    # If we get a script in bytes, we want to decode the actual commands it has
    # This method does so, and returns an object of the class Script (hence @classmethod)
    @classmethod
    def parse(cls, s):
        # get the length of the entire field
        length = read_varint(s)
        # initialize the cmds array
        cmds = []
        # initialize the number of bytes we've read to 0
        count = 0
        # loop until we've read length bytes
        while count < length:
            # get the current byte
            current = s.read(1)
            # increment the bytes we've read
            count += 1
            # convert the current byte to an integer
            current_byte = current[0]
            # if the current byte is between 1 and 75 inclusive
            if current_byte >= 1 and current_byte <= 75:
                # we have an cmd set n to be the current byte
                n = current_byte
                # add the next n bytes as an cmd
                cmds.append(s.read(n))
                # increase the count by n
                count += n
            # I'm omitting two cases that are currently supported in Bitcoin
            # We'll examine OP_PUSHDATA1 and OP_PUSHDATA2 only in theory :-)
            # I'll provide a more complete code later on
            else:
                # we have an opcode. set the current byte to op_code
                op_code = current_byte
                # add the op_code to the list of cmds
                cmds.append(op_code)
        if count != length:
            raise SyntaxError('parsing script failed')
        return cls(cmds)

    # This is the opposite of parse: we have an object of class Script, and we need to return bytes
    # Serializes the object of our class to a format accepted by the network
    def raw_serialize(self):
        # initialize what we'll send back
        result = b''
        # go through each cmd
        for cmd in self.cmds:
            # if the cmd is an integer, it's an opcode
            if type(cmd) == int:
                # turn the cmd into a single byte integer using int_to_little_endian
                result += int_to_little_endian(cmd, 1)
            else:
                # otherwise, this is an element
                # get the length in bytes
                length = len(cmd)
                # We'll assume at most 75 bytes of data
                if length >= 75:
                    raise ValueError('too long an cmd')
                    # Technically we should support up to 520 bytes; same as with OP_PUSHDATA1/2
                result += int_to_little_endian(length, 1)
                result += cmd
        return result

    # This just completes the job and wraps everything as needed
    def serialize(self):
        # get the raw serialization (no prepended length)
        result = self.raw_serialize()
        # get the length of the whole thing
        total = len(result)
        # encode_varint the total length of the result and prepend
        return encode_varint(total) + result

    # How do we check that the script is OK?
    # We need to evaluate it
    # Any checksig operation will have to verify a signature of the (hash of the) corresponding transaction
    # For this, we will pass this hash, z, as a parameter to the evaluation method
    def evaluate(self, z):
        # create a copy as we may need to add to this list if we have a RedeemScript
        cmds = self.cmds[:]
        stack = []
        # Technically, Script has an acces to a second stack via a few commands, but we will not use that here
        while len(cmds) > 0:
            cmd = cmds.pop(0)
            if type(cmd) == int:
                # The command is an pocode, so do what the opcode says
                operation = OP_CODE_FUNCTIONS[cmd]

                # The commands that will need the z for checking sig are OP_CHECKSIG and OP_CHECKSIGVERIFY
                # Their codes are 172 and 173
                # On https://en.bitcoin.it/wiki/Script you can also check that 174 and 175 also need z
                if cmd in (172, 173):
                    # these are signing operations, they need a sig_hash
                    # to check against
                    if not operation(stack, z):
                        return False
                else:
                    if not operation(stack):
                        return False
            else:
                # cmd is data, so add the cmd to the stack
                stack.append(cmd)

        # If we end up with an empty stack the execution failed
        if len(stack) == 0:
            return False
        # If we end up with False on the stack, the execution also failed
        # Need to tell you about numbers a bit more
        if stack.pop() == b'':
            return False
        return True

# A shortcut for creating a P2PKH script from the hash160 that appears in the script
def p2pkh_script(h160):
    '''Takes a hash160 and returns the p2pkh ScriptPubKey'''
    return Script([0x76, 0xa9, h160, 0x88, 0xac])

# A shortcut for creating a P2PKH script from the p2pkh address
def p2pkh_script_from_address(address):
    '''Takes a p2pkh address and returns the p2pkh ScriptPubKey for this address'''

    # The address format is what you would see on the network; e.g n3jKhCmVjvaVgg8C5P7E48fdRkQAAvf7Wc
    # THIS IS NOT IN BYTES!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

    # Check the prefix: https://en.bitcoin.it/wiki/List_of_address_prefixes
    if not ((address[0] == '1') or (address[0] == 'm') or (address[0] == 'n')):
        raise ValueError('not a valid p2pkh address')

    h160 = decode_base58(address)

    return Script([0x76, 0xa9, h160, 0x88, 0xac])

# A shortcut for creating a P2SH script from the hash160 that appears in the script
def p2sh_script(h160):
    '''Takes a hash160 and returns the p2sh ScriptPubKey'''
    return Script([0xa9, h160, 0x87])

# A shortcut for creating a P2SH script from the p2sh address that appears in the script
def p2sh_script_from_address(address):
    '''Takes a hash160 and returns the p2sh ScriptPubKey'''

    # The address format is what you would see on the network; e.g 3P14159f73E4gFr7JterCCQh9QjiTjiZrG

    # Check the prefix: https://en.bitcoin.it/wiki/List_of_address_prefixes
    if not ((address[0] == '3') or (address[0] == '2')):
        raise ValueError('not a valid p2sh address')

    h160 = decode_base58(address)

    return Script([0xa9, h160, 0x87])

##############################
###########TESTING############
##############################

'''
tmp = hash160(b'Hola')

### Script is: data data OP_EQUAL
script = Script([tmp, tmp, 0x87])

print(script.evaluate(tmp))

### Script is: data data OP_EQUALVERIFY
script = Script([tmp, tmp, 0x88])

print(script.evaluate(tmp))

##################################

z = 0x7c076ff316692a3d7eb3c3bb0f8b1488cf72e1afcd929e29307032997a838a3d
sec = bytes.fromhex('04887387e452b8eacc4acfde10d9aaf7f6d9a0f975aabb10d006e4da568744d06c61de6d95231cd89026e286df3b6ae4a894a3378e393e93a0f45b666329a0ae34')
sig = bytes.fromhex('3045022000eff69ef2b1bd93a66ed5219add4fb51e11a840f404876325a1e8ffe0529a2c022100c7207fee197d27c618aea621406f6bf5ef6fca38681d82b2f06fddbdce6feab601')

script = Script([sig, sec, 0xac])

print(script.evaluate(z))

###################################

# Generating my private key:
secret = hash256(b'IIC3272Sucks')
intSecret = int(secret.hex(),16)

privKey = PrivateKey(intSecret)

# Signing a message:
message = hash256(b'This course is boring!')
z = int(message.hex(),16)

signature = privKey.sign(z)

# Get the public key in SEC format
secPubKey = privKey.point.sec()

# Get the signature in DER format
derSig = signature.der()

# Need to append the SIGHASH_ALL field to verify the entire transaction!!!!!!!!!
derSig = derSig + SIGHASH_ALL.to_bytes(1,'big')

script = Script([derSig, secPubKey, 0xac])
print(script.evaluate(z))

###################################
'''
'''
########################
#########P2PKH##########
########################

# Generating my private key:
secret = hash256(b'IIC3272Sucks')
intSecret = int(secret.hex(),16)

privKey = PrivateKey(intSecret)

# Signing a message:
message = hash256(b'This course is boring!')
z = int(message.hex(),16)

signature = privKey.sign(z)

# Get the public key in SEC format
secPubKey = privKey.point.sec()

# Get the signature in DER format
derSig = signature.der()

# Need to append the SIGHASH_ALL field to verify the entire transaction!!!!!!!!!
derSig = derSig + SIGHASH_ALL.to_bytes(1,'big')

h160 = hash160(secPubKey)

script = Script([derSig, secPubKey, 0x76, 0xa9, h160, 0x88, 0xac])
print(script.evaluate(z))
'''
'''
# Generating my private key:
secret = hash256(b'IIC3272Sucks')
intSecret = int(secret.hex(),16)

privKey = PrivateKey(intSecret)

# Signing a message:
message = hash256(b'This course is boring!')
z = int(message.hex(),16)

signature = privKey.sign(z)

# Get the public key in SEC format
secPubKey = privKey.point.sec()

# Get the signature in DER format
derSig = signature.der()

# Need to append the SIGHASH_ALL field to verify the entire transaction!!!!!!!!!
derSig = derSig + SIGHASH_ALL.to_bytes(1,'big')

h160 = hash160(secPubKey)

print(h160)
print(h160.hex())

script = Script([0x76, 0xa9, h160, 0x88, 0xac])
print(script)
'''

'''
##################################################
# Generating a p2pkh script from a p2pkh address #
##################################################
address = 'n3jKhCmVjvaVgg8C5P7E48fdRkQAAvf7Wc'
print('The script:',p2pkh_script_from_address(address))
print('The script serialized in hex:',p2pkh_script_from_address(address).serialize().hex())

h160 = decode_base58(address)
print(h160_to_p2pkh_address(h160,testnet=True))

address = '2NGZrVvZG92qGYqzTLjCAewvPZ7JE8S8VxE'
print('The script:',p2sh_script_from_address(address))
print('The script serialized in hex:',p2sh_script_from_address(address).serialize().hex())

h160 = decode_base58(address)
print(h160_to_p2sh_address(h160,testnet=True))

'''


from darknet import darknet
import cv2, numpy as np, sys
from ctypes import *

net = darknet.load_net(b"D:/python_module/darknet-master/build/darknet/x64/project/myyolov3.cfg", b"D:/python_module/darknet-master/build/darknet/x64/project/backup/myyolov3_final.weights", 0)
meta = darknet.load_meta(b"D:/python_module/darknet-master/build/darknet/x64/project/my.data")
cap = cv2.VideoCapture("D:/python_module/darknet-master/build/darknet/x64/project/26-4.mp4")
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
fps = cap.get(cv2.CAP_PROP_FPS)
out = cv2.VideoWriter('otter_out26-4.avi', fourcc, fps, (640,480))
print(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
print(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
i = 0
while(cap.isOpened()):
    i += 1
    ret, image = cap.read()
    image = cv2.resize(image, dsize=(640, 480), interpolation=cv2.INTER_AREA)
    print(i)
    if not ret:
        break
    frame = darknet.nparray_to_image(image)
    r = darknet.detect_image(net, meta, frame, thresh=.5, hier_thresh=.5, nms=.45, debug= False)
    print(r)
    boxes = []

    for k in range(len(r)):
        width = r[k][2][2]
        height = r[k][2][3]
        center_x = r[k][2][0]
        center_y = r[k][2][1]
        bottomLeft_x = center_x - (width / 2)
        bottomLeft_y = center_y - (height / 2)
        x, y, w, h = bottomLeft_x, bottomLeft_y, width, height
        mytexts = r[k][0]
        mythresh = r[k][1]
        boxes.append((x, y, w, h, mytexts, mythresh))
    print(1)
    for k in range(len(boxes)):
        x, y, w, h, texts, threshs = boxes[k]
        top = max(0, np.floor(x + 0.5).astype(int))
        left = max(0, np.floor(y + 0.5).astype(int))
        right = min(image.shape[1], np.floor(x + w + 0.5).astype(int))
        bottom = min(image.shape[0], np.floor(y + h + 0.5).astype(int))
        cv2.rectangle(image, (top, left), (right, bottom), (255, 0, 0), 2)
        cv2.putText(image, texts.decode('utf-8') + '('+ str(threshs*100)[:5] + '%)', (top, left-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0))
        # cv2.line(image, (top + int(w / 2), left), (top + int(w / 2), left + int(h)), (0,255,0), 3)
        # cv2.line(image, (top, left + int(h / 2)), (top + int(w), left + int(h / 2)), (0,255,0), 3)
        # cv2.circle(image, (top + int(w / 2), left + int(h / 2)), 2, tuple((0,0,255)), 5)

    cv2.imshow('frame', image)
    out.write(image)
    darknet.free_image(frame) ## darknet에서 쓰는 c언어 동적할당 해제해주는 함수(써주어야 메모리가 버틴다.)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
out.release()
cv2.destroyAllWindows()


import jwt
from django.http.response import JsonResponse
from rest_framework import status
from config import config
import re
from enum import Enum

def jwt_check_middleware(get_response):
    def middleware(request):

        ignore_check_urls = [
            r'^(\/)$'
            r'^(\/member\/login)$'
            r'^(\/member\/register)$'
        ]

        for pattern in ignore_check_urls:
            if bool(re.search(pattern, request.path)):
                return get_response(request)
        try:
            auth = request.META.get('HTTP_AUTHORIZATION').split()
        except AttributeError:
            return JsonResponse({"message": JWTCheckErrorTypes.NO_AUTHENTICATE_HEADER.value},
                status = status.HTTP_401_UNAUTHORIZED)
        if auth[0].lower() == 'token':
            try:
                payload = jwt.decode(
                    auth[1], config['SECRET_KEY'], algorithms = ['HS256']

                )
                data = payload.get('data')
                request.META['TOKEN_PAYLOAD'] = data
                response = get_response(request)
                return response
            except jwt.ExpiredSignatureError:
                return JsonResponse({"message": JWTCheckErrorTypes.TOKEN_EXPIRED.value},
                status = status.HTTP_401_UNAUTHORIZED)
            except jwt.InvalidTokenError:
                return JsonResponse({"message": JWTCheckErrorTypes.INVALID_TOKEN.value},
                status = status.HTTP_401_UNAUTHORIZED)
        else:
            return JsonResponse({"message": JWTCheckErrorTypes.NOT_SUPPORT_AUTH_TYPE.value},
                status = status.HTTP_401_UNAUTHORIZED)
    return middleware

class JWTCheckErrorTypes(Enum):
    NO_AUTHENTICATE_HEADER = 'No authenticate header'
    NOT_SUPPORT_AUTH_TYPE = 'No support auth type'
    INVALID_TOKEN = 'Invalid token'
    TOKEN_EXPIRED = 'Token expired'


import os
from datetime import datetime
from decimal import Decimal

from influxdb import InfluxDBClient

from ..acoustics import acoustic_analysis

from ..sql.models import SoundFile, Discourse

from ..graph.discourse import DiscourseInspecter

def sanitize_formants(value):
    try:
        f1 = value['F1'][0]
    except TypeError:
        f1 = value['F1']
    if f1 is None:
        f1 = 0
    try:
        f2 = value['F2'][0]
    except TypeError:
        f2 = value['F2']
    if f2 is None:
        f2 = 0
    try:
        f3 = value['F3'][0]
    except TypeError:
        f3 = value['F3']
    if f3 is None:
        f3 = 0
    return f1, f2, f3

def to_nano(seconds):
    if not isinstance(seconds,Decimal):
        seconds = Decimal(seconds).quantize(Decimal('0.001'))
    return int(seconds * Decimal('1e9'))

def to_seconds(timestring):
    try:
        d = datetime.strptime(timestring, '%Y-%m-%dT%H:%M:%S.%fZ')
    except:
        d = datetime.strptime(timestring, '%Y-%m-%dT%H:%M:%SZ')

    s = 60*60*d.hour + 60*d.minute + d.second + d.microsecond / 1e6
    s = Decimal(s).quantize(Decimal('0.001'))
    return s

class AudioContext(object):
    """
    Class that contains methods for dealing with audio files for corpora
    """
    def analyze_acoustics(self, stop_check = None, call_back = None):
        """
        Runs all acoustic analyses for the corpus.
        """
        if not self.has_sound_files:
            raise(NoSoundFileError)
        acoustic_analysis(self, stop_check = stop_check, call_back = call_back)

    def genders(self):

        res = self.execute_cypher('''MATCH (s:Speaker:{corpus_name}) RETURN s.gender as gender'''.format(corpus_name = self.cypher_safe_name))
        genders = set()
        for s in res:
            g = s['gender']
            if g is None:
                g = ''
            genders.add(g)
        return sorted(genders)

    def reset_acoustics(self, call_back = None, stop_check = None):
        self.acoustic_client().drop_database(self.corpus_name)

    def acoustic_client(self):
        client = InfluxDBClient(**self.config.acoustic_conncetion_kwargs)
        databases = client.get_list_database()
        if self.corpus_name not in databases:
            client.create_database(self.corpus_name)
        return client

    def inspect_discourse(self, discourse, begin = None, end = None):
        """
        Get a discourse inspecter object for a discourse

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float, optional
            Beginning of the initial cache
        end : float, optional
            End of the initial cache

        Returns
        -------
        :class:`~polyglotdb.graph.discourse.DiscourseInspecter`
            DiscourseInspecter for the specified discourse
        """
        return DiscourseInspecter(self, discourse, begin, end)

    def discourse_sound_file(self, discourse):
        """
        Gets the sound file object for the discourse

        Parameters
        ----------
        discourse : str
            Name of the discourse

        Returns
        -------
        :class:`~polyglotdb.sql.models.SoundFile`
            the first soundfile
        """
        q = self.sql_session.query(SoundFile).join(SoundFile.discourse)
        q = q.filter(Discourse.name == discourse)
        sound_file = q.first()
        return sound_file

    def discourse_audio_directory(self, discourse):
        """
        Return the directory for the stored audio files for a discourse
        """
        return os.path.join(self.config.audio_dir, discourse)

    def has_all_sound_files(self):
        """
        Check whether all discourses have a sound file

        Returns
        -------
        bool
            True if a sound file exists for each discourse name in corpus,
            False otherwise
        """
        if self._has_all_sound_files is not None:
            return self._has_all_sound_files
        discourses = self.discourses
        for d in discourses:
            sf = self.discourse_sound_file(d)
            if sf is None:
                break
            if not os.path.exists(sf.filepath):
                break
        else:
            self._has_all_sound_files = True
        self._has_all_sound_files = False
        return self._has_all_sound_files

    @property
    def has_sound_files(self):
        """
        Check whether any discourses have a sound file

        Returns
        -------
        bool
            True if there are any sound files at all, false if there aren't
        """
        if self._has_sound_files is None:
            self._has_sound_files = self.sql_session.query(SoundFile).first() is not None
        return self._has_sound_files

    def get_formants(self, discourse, begin, end, channel = 0, source = None):
        """
        Get formants for a given discourse and time range

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float
            Beginning of time range
        end : float
            End of time range
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the formant_algorithm
            attribute of the CorpusContext

        Returns
        -------
        list
            List of results with fields for ``time``, ``F1``, ``F2``, and ``F3``
        """
        begin = Decimal(begin).quantize(Decimal('0.001'))
        end = Decimal(end).quantize(Decimal('0.001'))
        if source is None:
            source = self.config.formant_algorithm
        client = self.acoustic_client()
        result = client.query('''select "time", "F1", "F2", "F3" from "formants"
                        WHERE "discourse" = '{}' AND "source" = '{}'
                        AND "time" >= {}
                        AND "time" <= {};'''.format(discourse, source,
                                                to_nano(begin),to_nano(end)))
        listing = []
        for r in result.get_points('formants'):
            s = to_seconds(r['time'])
            listing.append((s, r['F1'], r['F2'], r['F3']))
        return listing

    def get_pitch(self, discourse, begin, end, channel = 0, source = None):
        """
        Get pitch for a given discourse and time range

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float
            Beginning of time range
        end : float
            End of time range
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the pitch_algorithm
            attribute of the CorpusContext

        Returns
        -------
        list
            List of results with fields for ``time`` and ``F0``
        """
        begin = Decimal(begin).quantize(Decimal('0.001'))
        end = Decimal(end).quantize(Decimal('0.001'))
        if source is None:
            source = self.config.pitch_algorithm
        client = self.acoustic_client()
        query = '''select "time", "F0" from "pitch"
                        WHERE "discourse" = '{}' AND "source" = '{}'
                        AND "time" >= {}
                        AND "time" <= {};'''.format(discourse, source,
                                                to_nano(begin),to_nano(end))
        result = client.query(query)
        listing = []
        for r in result.get_points('pitch'):
            s = to_seconds(r['time'])
            listing.append((s, r['F0']))
        return listing

    def save_formants(self, sound_file, formant_track, channel = 0, source = None):
        """
        Save a formant track for a sound file

        Parameters
        ----------
        sound_file : str or :class:`~polyglotdb.sql.models.SoundFile`
            Discourse name or SoundFile object
        formant_track : dict
            Dictionary with times as keys and tuples of F1, F2, and F3 values as values
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the formant_algorithm
            attribute of the CorpusContext
        """
        if isinstance(sound_file, str):
            sound_file = self.discourse_sound_file(sound_file)
        if sound_file is None:
           return
        if source is None:
            source = self.config.formant_algorithm
        data = []
        for timepoint, value in formant_track.items():
            f1, f2, f3 = sanitize_formants(value)
            d = {'measurement': 'formants',
                'tags': {'discourse': sound_file.discourse.name,
                        'channel': channel,
                        'source': source},
                "time": to_nano(timepoint),
                "fields": {'F1': f1,'F2':f2,'F3':f3}
                }
            data.append(d)
        self.acoustic_client().write_points(data)

    def save_pitch(self, sound_file, pitch_track, channel = None, speaker = None, source = None):
        """
        Save a pitch track for a sound file

        Parameters
        ----------
        sound_file : str or :class:`~polyglotdb.sql.models.SoundFile`
            Discourse name or SoundFile object
        pitch_track : dict
            Dictionary with times as keys and F0 values as values
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the pitch_algorithm
            attribute of the CorpusContext
        """
        if isinstance(sound_file, str):
            sound_file = self.discourse_sound_file(sound_file)
        if sound_file is None:
           return
        if source is None:
            source = self.config.pitch_algorithm
        data = []
        tag_dict = {}
        if isinstance(sound_file, str):
            tag_dict['discourse'] = sound_file
        else:
            tag_dict['discourse'] = sound_file.discourse.name
        if speaker is not None:
            tag_dict['speaker'] = speaker
        if source is not None:
            tag_dict['source'] = source
        if channel is not None:
            tag_dict['channel'] = channel
        for timepoint, value in pitch_track.items():
            try:
                value = float(value['F0'])
            except TypeError:
                try:
                    value = float(value[0])
                except TypeError:
                    try:
                        value = float(value)
                    except ValueError:
                        continue

            d = {'measurement': 'pitch',
                'tags': tag_dict,
                "time": to_nano(timepoint),
                "fields": {'F0': value}
                }
            data.append(d)
        self.acoustic_client().write_points(data)

    def has_formants(self, discourse, source = None):
        """
        Return whether a discourse has any formant values associated with it
        """
        client = self.acoustic_client()
        if source is None:
            source = self.config.pitch_algorithm
        result = client.query('''select "F1" from "formants" WHERE "discourse" = '{}' AND "source" = '{}' LIMIT 1;'''.format(discourse, source))
        if len(result) == 0:
            return False
        return True

    def has_pitch(self, discourse, source = None):
        """
        Return whether a discourse has any pitch values associated with it
        """
        client = self.acoustic_client()
        if source is None:
            source = self.config.pitch_algorithm
        result = client.query('''select "F0" from "pitch" WHERE "discourse" = '{}' AND "source" = '{}' LIMIT 1;'''.format(discourse, source))
        if len(result) == 0:
            return False
        return True


import os
import csv
import numpy as np
version = 6
n_pts = 10000
input_dir = f"/home/theresa/p/data_v{version}"
medians = []

for ff in os.listdir(input_dir):
    if ff != '.csv':
        filename = os.path.join(input_dir, ff)
        with open(filename, "r") as f:
            reader = csv.reader(f)
            dists = []
            for row in reader:
                data = [float(element) for element in row[:]]
                dist = float(row[-1])
                dists.append(dist)
            medians.append(np.median(np.array(dists)))
            # if (len(class_a + class_b)< n_pts):
            #     print("not enough points")
            #     continue
            # # make class a the smaller class
            # class_c = class_a
            # if len(class_b) < len(class_a):
            #     class_a = class_b
            #     class_b = class_c
            #
            # # sample half of each
            # class_a_n = min(len(class_a), n_pts / 2)
            # remainder = 0
            # if class_a_n < n_pts / 2:
            #     remainder = (n_pts / 2) - class_a_n
            # class_b_n = remainder + (n_pts / 2)
            # bidx = np.random.choice(len(class_b), int(class_b_n), replace=False)
            # aidx = np.random.choice(len(class_a), int(class_a_n), replace=False)
            # newa = [a  for idx, a in filter( lambda x:x[0] in aidx, list(enumerate(class_a)))]
            # newb = [b  for idx, b in filter( lambda x:x[0] in bidx, list(enumerate(class_b)))]
            # # print(len(newa + newb))
            # with open(outfile, "w") as o:
            #     reader = csv.writer(o)
            #     for row in newa + newb:
            #         reader.writerow(row)

print("Median threshold: ", np.median(np.array(medians)))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.shortcuts import render
from django.http import JsonResponse
from django.urls import reverse

from .tables import DashboardTable

def dashboard(request):
    context = dict(
        table_coins=DashboardTable(
            id='table_coins',
            url=reverse('api:coin-list', kwargs={'format': 'json'}),
            columns=['code', 'name']
        ),
        table_markets=DashboardTable(
            id='table_markets',
            url=reverse('api:market-list', kwargs={'format': 'json'}),
            columns=['code', 'coin', 'unit']
        ),
        table_tickers=DashboardTable(
            id='table_tickers',
            url=reverse('api:ticker-list', kwargs={'format': 'json'}),
            columns=['market', 'price', 'volumn', 'date', 'sequence']
        )
    )
    return render(request, 'index.html', context)

def check_queries(request):
    context = dict(
        table_check_queries=DashboardTable(
            id='table_check_queries',
            url=reverse('api:querytest-list', kwargs={'format': 'json'}),
            columns=['name', 'params', 'result', 'expect', 'created_at', 'passed']
        )
    )
    return render(request, 'check.html', context)


from django.core import serializers
import json

class Japi:
	"""API simple responder class"""

	response = { 'status': 'ok', 'action':'' }

	def __init__( self, action ):
		self.response[ 'action' ] = action
		if not hasattr( self, action ):
			self.response[ 'status' ] = "ko"
			self.response[ 'error'] = action + " not found!"
		else:
			func = getattr( self, action )
			func()

	def __str__( self ):
		return json.dumps( self.response )

	def help( self ):
		self.response[ 'available_methods' ] = {
			'help':'this action, provide some basic snippets'
		}


with open('text_files/text_file_example.txt') as file_object:
    contents=file_object.read()
    print(contents.rstrip())
with open("pi_digits.txt") as line_example:
    for line in line_example:
        print(line.rstrip())
with open("pi_digits.txt") as third_example:
    lines=third_example.readlines()
    print(lines)
    for line in lines:
        print(line)
pi_string=""
for line in lines:
    pi_string+=line.rstrip()+" "

print(pi_string)
print(len(pi_string))
birth_day=input("Input your bith day format ddmmyy : ")
if birth_day in pi_string.strip():
    print("Your birth day in pi number")
else:
    print("Your birthday not appears in pi number")


from dataclasses import dataclass
from enum import Enum
from typing import Optional, Tuple, Dict, Union, Any

@dataclass
class Link:
    url: str
    bookmark_id: str
    heading_id: str

def hex_to_rgb(input: str):
    if input.startswith("#"):
        input = input[1:]
    return tuple(int(input[i : i + 2], 16) / 255.0 for i in (0, 2, 4))

def doc_color_to_color(info: Optional[dict]):
    if not info:
        return None
    base = info.get("color")
    if not base:
        return None
    base = base.get("rgbColor")
    if not base:
        return
    return Color(red=base["red"], green=base["green"], blue=base["blue"])

def doc_link_to_link(info: dict):
    if not info:
        return None
    # TODO: identify link format for these:
    # info.get('bookmarkId'), info.get('headingId')
    return info.get("url")  # Link(info.get('url'), info.get('bookmarkId'), info.get('headingId'))

@dataclass
class Color:
    hex: Optional[str] = None
    red: Optional[float] = None
    blue: Optional[float] = None
    green: Optional[float] = None
    alpha: Optional[float] = None

    @classmethod
    def parse(cls, item: Any):
        if isinstance(item, Color):
            return item
        elif isinstance(item, str):
            return Color(item)
        elif isinstance(item, tuple):
            return Color(None, *item)
        elif not item:
            return None
        else:
            raise ValueError(f"Cannot parse {item} to Color")

    def __post_init__(self):

        assert self.hex or (
            self.red is not None and self.blue is not None and self.green is not None
        )
        if self.hex:
            self.red, self.green, self.blue = hex_to_rgb(self.hex)

    def to_json(self):

        return {"color": {"rgbColor": {"red": self.red, "green": self.green, "blue": self.blue}}}

    def to_sheet_style(self):

        base = {"red": self.red, "green": self.green, "blue": self.blue}

        if self.alpha is not None:
            base["alpha"] = self.alpha

        return base

    def to_slide_style(self):

        base = {"color": {"rgbColor": {"red": self.red, "green": self.green, "blue": self.blue}}}

        if self.alpha is not None:
            base["alpha"] = self.alpha

        return base

@dataclass
class TextStyle:
    font_size: Optional[int] = None
    font: Optional[str] = None
    font_weight: Optional[int] = None
    color: Optional[Color] = None
    background_color: Optional[Color] = None
    bold: Optional[bool] = None
    italic: Optional[bool] = None
    strikethrough: Optional[bool] = None
    small_caps: Optional[bool] = None
    underline: Optional[bool] = None
    link: Optional[str] = None
    font_unit: Optional[str] = "PT"

    def to_doc_style(self) -> Tuple[str, Dict]:  # noqa: C901
        base: Dict[str, Union[bool, str, int, dict]] = {}
        masks = []
        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.small_caps is not None:
            base["smallCaps"] = self.small_caps
            masks.append("smallCaps")
        if self.background_color is not None:
            base["backgroundColor"] = self.background_color.to_json()
            masks.append("backgroundColor")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_json()
            masks.append("foregroundColor")
        if self.font is not None:
            if not base.get("weightedFontFamily"):
                base["weightedFontFamily"] = {}
            base["weightedFontFamily"]["fontFamily"] = self.font  # type: ignore
            masks.append("weightedFontFamily.fontFamily")
        if self.font_weight is not None:
            if not base.get("weightedFontFamily"):
                base["weightedFontFamily"] = {}
                base["weightedFontFamily"]["fontFamily"] = "Arial"  # type: ignore
            base["weightedFontFamily"]["weight"] = self.font_weight  # type: ignore
            masks.append("weightedFontFamily.fontFamily")
            masks.append("weightedFontFamily.weight")

        return ",".join(masks), base

    def to_sheet_style(self) -> Tuple[list, Dict]:  # noqa: C901

        base: Dict[str, Union[str, bool, int, dict]] = {}
        masks = []

        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_sheet_style()
            masks.append("foregroundColor")
        if self.font is not None:
            base["fontFamily"] = self.font
            masks.append("fontFamily")

        return masks, base

    def to_slide_style(self):

        base = {}
        masks = []

        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_sheet_style()
            masks.append("foregroundColor")
        if self.font is not None:
            base["fontFamily"] = self.font
            masks.append("fontFamily")
        if self.link is not None:
            base["link"] = {"url": self.link}
            masks.append("link")
        return masks, base

    @classmethod
    def from_doc_style(cls, info):
        if info.get("namedStyleType"):
            return DefaultFonts[info.get("namedStyleType")]
        return TextStyle(
            font_size=info.get("fontSize", {}).get("magnitude"),
            font_unit=info.get("fontSize", {}).get("unit"),
            font=info.get("font"),
            font_weight=info.get("fonWeight"),
            color=doc_color_to_color(info.get("color")),
            background_color=doc_color_to_color(info.get("backgroundColor")),
            bold=info.get("bold"),
            italic=info.get("italic"),
            strikethrough=info.get("strikethrough"),
            small_caps=info.get("smallCaps"),
            underline=info.get("underline"),
            link=doc_link_to_link(info.get("link")),
        )

class BorderStyle(Enum):
    NONE = "NONE"
    DOTTED = "DOTTED"
    DASHED = "DASHED"
    SOLID = "SOLID"
    SOLID_MEDIUM = "SOLID_MEDIUM"
    SOLID_THICK = "SOLID_THICK"
    DOUBLE = "DOUBLE"

class BorderPosition(Enum):
    TOP = "top"
    BOTTOM = "bottom"
    LEFT = "left"
    RIGHT = "right"
    INNER_HORIZONTAL = "innerHorizontal"
    INNER_VERTICAL = "innerVertical"

@dataclass
class Border:
    """Dataclass to represent a Border object for a border along a cell.

    Google documentation: https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/cells#Border
    """

    BORDER_STYLES = ["NONE", "DOTTED", "DASHED", "SOLID", "SOLID_MEDIUM", "SOLID_THICK", "DOUBLE"]
    COLOR_STYLES = ["rgbColor", "themeColor"]
    THEME_COLOR_TYPES = [
        "TEXT",
        "BACKGROUND",
        "ACCENT1",
        "ACCENT2",
        "ACCENT3",
        "ACCENT4",
        "ACCENT5",
        "ACCENT6",
        "LINK",
    ]

    position: BorderPosition
    style: BorderStyle
    color: Color
    color_style: Optional[Union[str, Color]] = None

    def to_json(self):

        # assert self.style in self.BORDER_STYLES

        base = {"style": self.style.value, "color": self.color.to_sheet_style()}

        if self.color_style is not None:
            assert self.color_style in self.COLOR_STYLES
            if self.color_style == "rgbColor":
                assert isinstance(self.color_style, Color)
                base["colorStyle"] = self.color_style.to_sheet_style()
            elif self.color_style == "themeColor":
                assert self.color_style in self.THEME_COLOR_TYPES
                base["themeColor"] = self.color_style

        return base

class DefaultFonts(Enum):
    NORMAL_TEXT = TextStyle(font_size=11, font="arial", color=Color(hex="#000000"))
    TITLE = TextStyle(font_size=26, font="arial", color=Color(hex="#000000"))
    SUBTITLE = TextStyle(font_size=15, font="arial", color=Color(hex="#666666"))
    HEADING1 = TextStyle(font_size=20, font="arial", color=Color(hex="#000000"))
    HEADING2 = TextStyle(font_size=16, font="arial", color=Color(hex="#666666"))
    HEADING3 = TextStyle(font_size=14, font="arial", color=Color(hex="#434343"))
    HEADING4 = TextStyle(font_size=12, font="arial", color=Color(hex="#666666"))
    HEADING5 = TextStyle(font_size=11, font="arial", color=Color(hex="#666666"))
    HEADING6 = TextStyle(font_size=11, font="arial", italic=True, color=Color(hex="#666666"))

    def to_doc_style(self):
        return self.value.to_doc_style()

@dataclass
class ParagraphStyle:
    alignment: str
    line_space: int
    direction: str
    spacing_mode: str
    space_above: str
    space_below: str
    border_between: str
    border_top: BorderStyle
    keep_lines_together: bool
    keep_with_next: bool
    avoid_widow_and_orphan: bool
    shading: str

    def to_doc_style(self) -> Tuple[str, Dict]:  # noqa: C901
        base = {}
        masks = []
        if self.alignment is not None:
            base["alignment"] = {"alignment": self.alignment}
            masks.append("alignment")
        # TODO: validate all of these
        for attr in [
            "line_space",
            "direction",
            "spacing_mode",
            "space_above",
            "space_below",
            "border_between",
            "border_top",
            "keep_lines",
            "keep_with_next",
            "avoid_widow_and_orphan",
            "shading",
        ]:
            test = getattr(self, attr)
            if test:
                components = attr.split("_")
                name = components[0] + "".join([component.title() for component in components])
                base[name] = {name: test}
                masks.append(name)
        return ",".join(masks), base

    def to_sheet_style(self) -> Tuple[list, Dict]:  # noqa: C901
        raise NotImplementedError

    @classmethod
    def from_doc_style(cls, info):
        if info.get("namedStyleType"):
            return DefaultFonts[info.get("namedStyleType")]
        return ParagraphStyle(**info)


from rest_framework import serializers
from stands.models import Stand
from events.models import Event
from events.serializers import EventListSerializer

class StandListSerializer(serializers.ModelSerializer):
    class Meta:
        model = Stand
        fields = ('person', 'emotion', 'age', 'sex', 'people', 'description', 'id', 'eventId')

class StandOwnerListSerializer(serializers.ModelSerializer):
    eventId = EventListSerializer()

    class Meta:
        model = Stand
        fields = '__all__'

class StandDetailSerializer(serializers.ModelSerializer):
    class Meta:
        model = Stand
        fields = '__all__'

class ImageSerializer(serializers.Serializer):
    image = serializers.CharField()
    stand = serializers.PrimaryKeyRelatedField(queryset=Stand.objects.all())


import argparse
import re
import threading
import time
from datetime import datetime

import numpy as np
import tensorflow as tf

if hasattr(tf, 'enable_eager_execution'):
    tf.enable_eager_execution()

from waymo_open_dataset.utils import frame_utils
from waymo_open_dataset import dataset_pb2 as open_dataset

from custom_utils import poolingOverlap, projected_points_to_image, turbo_rgba
import matplotlib.pyplot as plt
from custom_utils import colorize_np_arr

import errno
import os

from concurrent.futures import ThreadPoolExecutor, as_completed

executor = None

camera_names = {
    'front': open_dataset.CameraName.FRONT,
    # 'front_left': open_dataset.CameraName.FRONT_LEFT,
    # 'front_right': open_dataset.CameraName.FRONT_RIGHT,
    # 'side_left': open_dataset.CameraName.SIDE_LEFT,
    # 'side_right': open_dataset.CameraName.SIDE_RIGHT
}

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python ≥ 2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get(object_list, name):
    """ Search for an object by name in an object list. """

    object_list = [obj for obj in object_list if obj.name == name]
    return object_list[0]

def depth_map_by_camera_name(frame, camera, points_all, cp_points_all):
    """

    :param frame: frame camera
    :param camera: camera name
    :param points_all:  ndarray
    :param cp_points_all: ndarray
    :return:
    """
    camera_image = get(frame.images, camera)
    # get the image
    img_np = np.zeros(shape=(1280, 1920))
    # img_tensor = tf.image.decode_jpeg(camera_image.image)
    #
    # img_np = np.zeros_like(img_tensor.numpy())[:, :, 0].astype(np.float32)
    # img_np[:] = np.nan

    # The distance between lidar points and vehicle frame origin.
    points_all_tensor = tf.norm(points_all, axis=-1, keepdims=True)
    cp_points_all_tensor = tf.constant(cp_points_all, dtype=tf.int32)

    mask = tf.equal(cp_points_all_tensor[..., 0], camera_image.name)
    cp_points_all_tensor = tf.cast(
        tf.gather_nd(
            cp_points_all_tensor, tf.where(mask)), dtype=tf.float32)
    points_all_tensor = tf.gather_nd(points_all_tensor, tf.where(mask))

    projected_points_all_from_raw_data = tf.concat(
        [cp_points_all_tensor[..., 1:3], points_all_tensor],
        axis=-1).numpy()

    coordinates = projected_points_all_from_raw_data[:, 0:2]. \
        astype(np.int32)

    img_np[coordinates[:, 1], coordinates[:, 0]] = \
        projected_points_all_from_raw_data[:, 2]

    dense_matrix = \
        poolingOverlap(img_np, ksize=(15, 10), stride=(1, 1), method=None,
                       pad=True)

    return dense_matrix

def pool_points_project_to_image(dense_matrix, img_np):

    # TODO: this is super costly
    (width, height) = img_np.shape
    dense_matrix = np.nanmean(dense_matrix, axis=(2, 3))

    xv, yv = np.meshgrid(range(0, width), range(0, height), indexing='ij')
    xv, yv = xv.reshape(-1, ), yv.reshape(-1, )
    dense_vector = dense_matrix.reshape(-1, )
    valid_coordinates = dense_vector > 0.
    xv, yv = xv[valid_coordinates], yv[valid_coordinates]
    dense_vector = dense_vector[valid_coordinates]
    projected_matrix = np.vstack([yv, xv, dense_vector]).transpose()

    # project the points to an depth_np_arr
    depth_np_arr = projected_points_to_image(
        img_np.shape, projected_matrix
    )

    return depth_np_arr

def project_image_lidar_to_image_plane(frame, camera_names):
    (range_images, camera_projections, range_image_top_pose) = \
        frame_utils.parse_range_image_and_camera_projection(frame)
    # # convert the points to a point cloud from the first laser return
    points, cp_points = frame_utils.convert_range_image_to_point_cloud(
        frame, range_images, camera_projections, range_image_top_pose,
        ri_index=0)
    # ... and the second return
    points_second, cp_points_second = frame_utils.convert_range_image_to_point_cloud(
        frame, range_images, camera_projections, range_image_top_pose,
        ri_index=1)
    #
    points_all = np.concatenate(points, axis=0)
    cp_points_all = np.concatenate(cp_points, axis=0)
    points_second_all = np.concatenate(points_second, axis=0)
    cp_points_second_all = np.concatenate(cp_points_second, axis=0)
    #
    points_all = np.concatenate([points_all, points_second_all])
    cp_points_all = np.concatenate([cp_points_all, cp_points_second_all])

    # go through all the camera images
    _image_results = {}
    for camera_name, camera_dataset in camera_names.items():
        # img_np = np.zeros(shape=(1280, 1920))
        depth_np_matrix = depth_map_by_camera_name(
            frame, camera_dataset, points_all, cp_points_all
        )

        if camera_name not in _image_results:
            _image_results[camera_name] = depth_np_matrix
        else:
            raise BaseException('Camera-Name appear twice {}'.format(camera_name))

    return _image_results

def colorize_depth_map_save_img(depth_map, output_path, frame_id):
    print('[{}] {} Start colorizing: {}'.format(
        datetime.now(), threading.get_ident(), os.path.join(
            output_path, 'frame_{:04d}'.format(frame_id))
    ))
    img_np = np.zeros(shape=(1280, 1920))
    depth_np_arr = pool_points_project_to_image(depth_map, img_np)
    print(np.max(depth_np_arr))
    colorized_depth_image = colorize_np_arr(
        depth_np_arr, turbo_rgba
    )
    _save_path = os.path.join(output_path,
                              'frame_{:05d}.png'.format(frame_id))
    colorized_depth_image.save(_save_path)

def handle_frame(frame, output_data_folder, frame_id):
    print('[{}] {} Start handling frame'.format(
        datetime.now(),
        threading.get_ident(),
        os.path.join(output_data_folder, 'frame_{:04d}'.format(frame_id))))
    _s = datetime.now()
    lidar = project_image_lidar_to_image_plane(frame, camera_names)

    for camera_name, depth_np_matrix in lidar.items():
        _fpath = os.path.join(output_data_folder, camera_name)
        if not os.path.exists(_fpath):
            mkdir_p(_fpath)

        colorize_depth_map_save_img(depth_np_matrix, _fpath, frame_id)
        # future = executor.submit(
        #     colorize_depth_map_save_img, depth_np_matrix, _fpath, frame_id
        # )
        # futures.append(future)

    print('[{}] {} End handling frame'.format(datetime.now() - _s, threading.get_ident()))

def handle_file(input_filename, output_dir):
    assert executor is not None, 'executor has to be set to None'
    input_path, input_fname = os.path.split(input_filename)
    segment_m = re.search(
        '(.+_with_camera_labels).tfrecord$', input_fname
    )
    if segment_m is None:
        return
    # create the output folder
    if not os.path.exists(os.path.join(output_dir, 'lidar_rgb')):
        os.mkdir(os.path.join(output_dir, 'lidar_rgb'))

    output_data_folder = os.path.join(output_dir, 'lidar_rgb',
                                      segment_m.group(1))
    if not os.path.exists(output_data_folder):
        os.mkdir(output_data_folder)

    try:
        dataset = tf.data.TFRecordDataset(input_filename, compression_type='')
    except BaseException as e:
        print('Error while reading file: {}'.format(e))
        return

    read_frames = []
    print('[{}] Reading all frames'.format(datetime.now()))
    _s = datetime.now()
    for data in dataset:
        frame = open_dataset.Frame()
        try:
            frame.ParseFromString(bytearray(data.numpy()))
        except BaseException as e:
            print('\tError: {}'.format(e))
        read_frames.append(frame)
    print('[{}] Read all frames. Took {}'.format(
        datetime.now(), datetime.now() - _s))

    # # TODO: dropping frames for development purposes
    # read_frames = read_frames[:5]

    print('[{}] Extract dense matrices for all frames'.format(datetime.now()))
    _s = datetime.now()
    image_matrices = []
    futures = []

    for frame_id, frame in enumerate(read_frames):
        _f = executor.submit(handle_frame, frame, output_data_folder, frame_id)
        futures.append(_f)

    #
    #
    #
    # for i, frame in enumerate(read_frames):
    #     if i % 10 == 0:
    #         print('Frame {}/{}'.format(i, len(read_frames)))
    #         continue
    #     image_matrices.append(project_image_lidar_to_image_plane(frame, camera_names))
    # print('[{}] Extracted all dense-matrices for all frames ({} / sample)'.format(
    #     datetime.now() - _s, (datetime.now() - _s) / len(read_frames)))
    #
    # print('[{}] Colorize all frames'.format(datetime.now()))
    # futures = []
    # for frame_id, image_dict in enumerate(image_matrices):
    #     for camera_name, depth_np_matrix in image_dict.items():
    #         _fpath = os.path.join(output_data_folder, camera_name)
    #         if not os.path.exists(_fpath):
    #             mkdir_p(_fpath)
    #
    #         # colorize_depth_map_save_img(depth_np_matrix, _fpath, frame_id)
    #         future = executor.submit(
    #             colorize_depth_map_save_img, depth_np_matrix, _fpath, frame_id
    #         )
    #         futures.append(future)
    #
    # print('[{}] Colorized all frames'.format(datetime.now() - _s))

    return futures

if __name__ == '__main__':
    parser = argparse.ArgumentParser('export_data')
    parser.add_argument('--input_dir', type=str, help='Input directory')
    parser.add_argument('--output_dir', type=str, help='Output directory',
                        default='./output')
    parser.add_argument('--max_workers', type=int, default=1)
    parser.add_argument('--files_to_handle', type=int, default=-1)
    args = parser.parse_args()

    max_workers = 1
    if args.max_workers != -1:
        max_workers = args.max_workers
    executor = ThreadPoolExecutor(max_workers=max_workers)

    from turbo_map import RGBToPyCmap, turbo_colormap_data

    mpl_data = RGBToPyCmap(turbo_colormap_data)
    plt.register_cmap(name='turbo', data=mpl_data,
                      lut=turbo_colormap_data.shape[0])

    mpl_data_r = RGBToPyCmap(turbo_colormap_data[::-1, :])
    plt.register_cmap(name='turbo_r', data=mpl_data_r,
                      lut=turbo_colormap_data.shape[0])

    if not os.path.exists(args.output_dir):
        print('Output directory does not exist')
        exit(1)

    input_directory = os.path.join(
        os.path.expanduser('~'), 'Downloads', 'waymodataset'
    ) if args.input_dir is None else args.input_dir
    if not os.path.exists(input_directory):
        print('Input directory does not exist')
        exit(1)

    # get the files in the directory
    files = sorted(os.listdir(input_directory))
    if args.files_to_handle > 0:
        files = files[:args.files_to_handle]

    num_files = len(files)
    print(f'Reading from {input_directory} found {num_files} files')

    # go through them
    collected_futures = {}
    for i, f in enumerate(files):
        print(f'({i}/{num_files}) Extracting lidar data from {f}')
        _input_filename = os.path.join(input_directory, f)
        futures = handle_file(_input_filename, args.output_dir)
        collected_futures[f] = futures

    for key, futures in collected_futures.items():
        print(key, sum([f.done() for f in futures]))

    executor.shutdown(wait=True)


import logging
import time

from useful.resource.readers import ResourceURL, readers
from useful.resource.parsers import parsers

_log = logging.getLogger(__name__)

def cached_load(timeout=300):
    """
    Define timeout to be used in `load()` function.

    Args:
        timeout (int, optional): Number of seconds to cache data without
            checking if it has changed in any way. Defaults to 300.

    Returns:
        function: A function using timeout variable
    """
    memory = {}

    def load(url, mimetype=None, parser=None, hook=None):
        """
        Load resource from uri or cache if already used before.

        Args:
            url (str): String represeting URL specified in RFC 1738.
            mimetype (str, optional): Forced MIME type if not None. Defaults to
                None.
            parser (useful.resource.parsers.Parser, optional): A parser class
                to use instead of parsers from useful.resource.parsers.parsers.
                Defaults to None.
            hook (callable, optional): An optional function to call after
                reading and parsing the data. Defaults to None.

        Raises:
            ValueError: No reader supports provided url scheme
            ValueError: No parser supports provided mimetype

        Returns:
            object: Final data after running reader, parser and hook on the
                resource url
        """
        hash_ = None

        # get the reader from url
        resource_url = ResourceURL(url, mimetype=mimetype)
        try:
            reader = readers[resource_url.scheme]
        except KeyError:
            raise ValueError(
                f"Unsupported reader scheme '{resource_url.scheme}'")

        reader = reader(url=resource_url)

        # if url has been cached for less than `timeout` or hash sum of the
        # resource is still equal, return cached value
        if url in memory:
            now = time.time()
            if now - memory[url]['time'] < timeout:
                _log.debug(
                    f"Url '{url}' in memory for less then {timeout} seconds",
                    extra={"url": url, "timeout": timeout})
                return memory[url]['data']
            else:
                hash_ = reader.hash()
                if hash_ == memory[url]['hash']:
                    _log.debug(
                        f"Url '{url}' in memory hasn't changed hash sum",
                        extra={"url": url, "hash": hash_})
                    # update object timestamp in memory
                    memory[url]['time'] = now
                    return memory[url]['data']

        # if url has been cached but needs to update use cached hook as a
        # hook, otherwise use function parameter hook as hook object
        hook = memory.get(url, {}).get("hook", hook)
        # use already calculated above hash sum or calculate hash sum if it was
        # never calculated
        hash_ = hash_ or reader.hash()

        # use user-provided parser or get parser from mimetype
        try:
            parser = parser or parsers[resource_url.mimetype]
            parser = parser(reader=reader)
        except KeyError:
            raise ValueError(
                f"Unsupported parser mimetype {resource_url.mimetype}")

        # parse data provided by reader
        data = parser.parse()

        # call hook on data
        if hook is not None:
            data = hook(data)

        # cache results and other relevant data
        memory[url] = {
            'time': time.time(),
            'hash': hash_,
            'data': data,
            'hook': hook
        }
        _log.debug(f"Upserting url '{url}' in memory",
                   extra={"url": url, "hash": hash_})
        return data

    return load

load = cached_load(timeout=300)


import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import MeanShift
from sklearn.cluster import KMeans
from astroML.stats import binned_statistic_2d
FileType='NGC'
num='0821'

data=u'../DATA/%s%s.csv' %(FileType,num)
ID,RA,Dec,Magg,Errg,Magr,Errr=np.loadtxt(data, delimiter=',', usecols=(0,1,2,3,4,5,6), unpack=True)
ra=list()
dec=list()
for i in range(len(RA)):
    if Errr[i]<0.1 and Errg[i]<0.1:
        ra.append(RA[i])
        dec.append(Dec[i])

RA_range=[min(RA),max(RA)]
Dec_range=[min(Dec),max(Dec)]

ra_=np.array(ra).reshape(1,len(ra)).T
dec_=np.array(dec).reshape(1,len(dec)).T
X=np.hstack((ra_,dec_))
ms=MeanShift(bandwidth=0.4,cluster_all=False)
ms.fit(X)
centers=ms.cluster_centers_

N, xedges, yedges = binned_statistic_2d(ra, dec, ra ,'count', bins=30, range=[RA_range,Dec_range])
plt.figure(figsize=(9,9))
plt.imshow(N.T,origin='lower',extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]],cmap = plt.cm.gray_r,interpolation='gaussian')
plt.scatter(centers[:,0],centers[:,1])
plt.xlim(RA_range[1],RA_range[0])
plt.ylim(Dec_range)
plt.show()