from flair.data import Sentence
from flair.models import SequenceTagger
from flair.embeddings import (DocumentPoolEmbeddings,
                              FlairEmbeddings, StackedEmbeddings)

from torch import dot, norm, squeeze

from pathlib import Path
import pickle

from odm.nlp.tensors import PCA, plot_embeddings, normalize_tensors
from odm.nlp.parse import parse_veclang

class Model:

    modelpath = '/data/model/'

    def __init__(self):

        # Sequence Tagging Model
        tagger_file = self.modelpath + 'tagger.pt'
        if Path(tagger_file).is_file():
            print('loading tagger from file')
            self.tagger = SequenceTagger.load_from_file(tagger_file)
        else:
            print('downloading pretrained tagger')
            self.tagger = SequenceTagger.load('ner-ontonotes')
            self.tagger.save(tagger_file)

        # Text Embedding Model
        embeddings_file = self.modelpath + 'embeddings.pickle'
        if Path(embeddings_file).is_file():
            print('loading embedder from file')
            filestream = open(embeddings_file, 'rb')
            self.embeddings = pickle.load(filestream)
        else:
            print('downloading pretrained embedders')
            self.embeddings = [
                # WordEmbeddings('glove'),
                FlairEmbeddings('multi-forward')
                # FlairEmbeddings('multi-backward')
            ]
            filestream = open(embeddings_file, 'wb')
            pickle.dump(self.embeddings, filestream)

        self.token_embedder = StackedEmbeddings(self.embeddings)
        self.doc_embedder = DocumentPoolEmbeddings(self.embeddings)

    def parse(self, text):

        sentence = Sentence(text)
        self.tagger.predict(sentence)
        self.token_embedder.embed(sentence)
        self.doc_embedder.embed(sentence)

        return sentence

    def mindmap(self, text):

        # parsing
        lines, arrows = parse_veclang(text)
        sentences = [self.parse(line) for line in lines]
        tensors = [s.get_embedding() for s in sentences]

        # tensor processing
        norm_tensors = normalize_tensors(tensors)
        flat_tensors = PCA(norm_tensors)

        # plot map
        filename = plot_embeddings(lines, flat_tensors, arrows)

        return f'Plotted mindmap to {filename}'

    def similarity(self, text):
        lines = text.split('//')
        sentences = [self.parse(line) for line in lines]
        vecs = [squeeze(s.embedding) for s in sentences]
        sim = dot(vecs[0], vecs[1])/(norm(vecs[0])*norm(vecs[1]))

        return f'the similarity is {sim}'


#!/usr/bin/env python3
# 2次元累積和 S の [x1, x2) × [y1, y2) 総和
def ac2(s, x1, x2, y1, y2):
    return s[x2][y2] - s[x1][y2] - s[x2][y1] + s[x1][y1]

import numpy as np
_, *d = open(0)
n, k = map(int, _.split())
B = np.zeros((2*k, 2*k))
for e in d:
    *z, c = e.split()
    x, y = map(int, z)
    B[x % (2*k)][(y + k * (z == "W")) % (2*k)] += 1
B.cumsum(axis = 0)
B.cumsum(axis = 1)
B = np.tile(B, (2,2))
print(B)
# 書きかけ


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Resize and add text to videos

Usage example:
  $ python transform.py

"""

import random
import os
import shutil
from moviepy.editor import *
from moviepy.video.tools.subtitles import SubtitlesClip
from sqlite import *

class VideoTransform:
    def __init__(self, videos=[]):
        print(' ----> {} videos to transform'.format(len(videos)))
        for video in videos:
            video['clip'] = VideoFileClip(video['filepath'])
        self.videos = videos
        print(' --> {} videos converted to clips'.format(len(self.videos)))

    def adjustSize(self, clip,width,height):
        # get current dimensions and ratio
        clipWidth,clipHeight = clip.size
        ratio = clipWidth / clipHeight
        # resize the clip
        if ratio > (width/height):
            clip2 = clip.resize(width=width)
        else:
            clip2 = clip.resize(height=height)
        return clip2

    def adjustSizes(self,width=1920,height=1080):
        for video in self.videos:
            video['clip'] = self.adjustSize(video['clip'],width,height)

    def addSubtitle(self, clip, subtitle, howLong, maxLength=15):
        # trim the subtitle if it's too long
        if len(subtitle) > maxLength:
            subtitle = subtitle[:maxLength] + '...'
        # set how long the subtitle will show
        howLong = min(howLong,clip.duration)
        subs = [((0, howLong), subtitle)]
        # create clip with subtitle
        subtitles = SubtitlesClip(subs, lambda txt: TextClip(txt, font='Arial', fontsize=72, color='white', stroke_color='black'))
        # merge clip and subtitles
        result = CompositeVideoClip([clip, subtitles.set_position(('center','bottom'))])
        # return resulting clip
        return result

    def addSubtitles(self, howLong=10):
        print(' ----> Adding subtitles')
        for index, video in enumerate(self.videos):
            video['clip'] = self.addSubtitle(video['clip'],video['title'],howLong)
            print(' --> {}/{} subtitles added'.format(index+1,len(self.videos)))

    def addSource(self,clip,source,howLong):
        howLong = min(howLong,clip.duration)
        subs = [((0, howLong), source)]
        # create clip with source
        subtitles = SubtitlesClip(subs, lambda txt: TextClip(txt, font='Arial', fontsize=24, color='white', bg_color='black'))
        # merge clip and text
        result = CompositeVideoClip([clip, subtitles.set_position((0,0))])
        # return resulting clip
        return result

    def addSources(self, howLong=10):
        print(' ----> Adding sources')
        for index, video in enumerate(self.videos):
            video['clip'] = self.addSource(video['clip'],video['link'],howLong)
            print(' --> {}/{} sources added'.format(index+1,len(self.videos)))

    def save(self, clip, filename, threads=1):
        filepath = os.path.join(os.getcwd(),filename)
        clip.write_videofile(filepath, codec='mpeg4', audio=True, threads=threads)
        return filepath

if __name__ == '__main__':
    # get videos from DB
    sql = Sqlite()
    result = sql.run('SELECT * FROM videos WHERE duration IS NULL')
    videos = [ row for row in result if os.path.isfile(row['filepath']) ]

    # convert them to clips
    limit = 10
    transform = VideoTransform(videos[:limit])

    # resize them
    transform.adjustSizes()

    # add subtitles
    transform.addSubtitles()

    # add link to original
    transform.addSources()

    # backup the original videos
    # and save the updated versions
    for video in transform.videos:
        shutil.move(video['filepath'],video['filepath']+'.original')
        transform.save(video['clip'],video['filepath'])

    # update their durations and sizes on the DB
    for video in transform.videos:
        filepath = video['filepath']
        duration = video['clip'].duration
        width,height = video['clip'].size
        sql.run('''
            UPDATE videos
            SET duration = {duration},
                width    = {width},
                height   = {height}
            WHERE
                filepath = '{filepath}'
        '''.format(
            duration=duration,
            filepath=filepath,
            width=width,
            height=height
        ))


#!/usr/bin/python3
"""This is a module for the island func
"""

def island_perimeter(grid):
    """This is a function that returns perimeter
    of the island:
    traverse the grid to find a 1
    and then check all four directions
    """
    peri = 0
    xrows = grid

    if grid != []:
        ycols = grid[0]

    for x in range(len(xrows)):
        for y in range(len(ycols)):
            if grid[x][y] == 1:

                # check the left
                if y == 0 or grid[x][y - 1] == 0:
                    peri += 1

                # check the top
                if x == 0 or grid[x - 1][y] == 0:
                    peri += 1

                # check the right
                if y == len(ycols) - 1 or grid[x][y + 1] == 0:
                    peri += 1

                # check the bottom
                if x == len(xrows) - 1 or grid[x + 1][y] == 0:
                    peri += 1

    return peri


from globals import *

class MainHandler(webocrat_Request):

    @need_registered_user
    def get(self):
        template_vals = {}
        self.render_simple_template('HomePage.django.html', template_vals)

class HartaCainilorHandler(webocrat_Request):

#    @need_registered_user
    def get(self):
        template_vals = {
            'login_iframe': LOGIN_IFRAME,
            'BASE_URL': BASE_URL,
            'lat' : '44.44',
            'lng' : '26.1',
            'zoom' : 14
        }
        self.session['redirect_after_login'] = self.request.url
        self.render_template('HartaCainilor.django.html', template_vals)

from webapp2_extras import routes

app = webapp2.WSGIApplication([
        routes.DomainRoute('localhost',
            [webapp2.Route('/hartacainilor', handler=HartaCainilorHandler, name='hartacainilor-home0'),
        ]),
        routes.DomainRoute('hartacainilor.webocrat.com',
            [webapp2.Route('/', handler=HartaCainilorHandler, name='hartacainilor-home'),
        ]),
        routes.DomainRoute('www.webocrat.com',
            [webapp2.Route('/hartacainilor', handler=HartaCainilorHandler, name='hartacainilor-home'),
        ]),
        routes.DomainRoute('www.webocrat.com',
            [webapp2.Route('/', handler=MainHandler, name="webocrat-home"),
        ]),
        routes.DomainRoute(BASE_URL,
            [webapp2.Route('/', handler=MainHandler, name="webocrat-home2"),
        ]),
      ])


import os

_basedir = os.path.abspath(os.path.dirname(__file__))

DEBUG = True

ADMINS = frozenset(['volodymyr.dehtiarenko@gmail.com'])
SECRET_KEY = 'This string will be replaced with a proper key in production.'

SQLALCHEMY_ECHO = True
SQLALCHEMY_DATABASE_URI = "mysql+pymysql://admin:12345@localhost/24servis"
SQLALCHEMY_TRACK_MODIFICATIONS = False

WTF_CSRF_ENABLED = True
WTF_CSRF_SECRET_KEY = "somethingimpossibletoguess"


import msgpack
import redis
import click
import socket
import pandas as pd
import numpy as np
import ujson as json

#Extra Functions (for enrichment purpose)
df_port_name = pd.read_csv('enrichments/port_name.txt',delimiter=",", names=['port_num','port_name'])
df_ip_proto_name = pd.read_csv('enrichments/ip_proto_name.txt',delimiter=",", names=['proto_num','proto_name'])

@click.group()
def cli():
    pass

@click.command()
def receive():
    r = redis.StrictRedis(host='localhost', port=6379, db=0)
    p = r.pubsub()
    p.subscribe('patterns')
    try:
        # non-blocking
        # while True:
        #     print(p.get_message())

        # blocking
        for message in p.listen():
            if isinstance(message['data'], bytes):
                payload = msgpack.unpackb(message['data'], encoding='utf-8')
                print(payload)
    except KeyboardInterrupt as e:
        print('Keyboard interrupt')

def get_ip_proto_name(ip_proto_number):
    try:
        return df_ip_proto_name[df_ip_proto_name['proto_num']==ip_proto_number]['proto_name'].values[0]
    except:
        return str(ip_proto_number)

def get_port_name(port_number):
    try:
        return df_port_name[df_port_name['port_num']==port_number]['port_name'].values[0]+" service port"
    except:
        return "port "+str(int(port_number))

def get_tcp_flag_name(tcp_flags_str):
    tcp_flags=""
    try:
        tcp_flags += ("FIN" if (tcp_flags_str.find('F') != -1) else next)
    except:
        next
    try:
        tcp_flags += ("SYN" if (tcp_flags_str.find('S')!= -1) else next)
    except:
        next

    try:
        tcp_flags += ("RST" if tcp_flags_str.find('R') != -1 else next)
    except:
        next

    try:
        tcp_flags += ("PUSH" if tcp_flags_str.find('P') != -1 else next)
    except:
        next

    try:
        tcp_flags += ("ACK" if tcp_flags_str.find('A') != -1 else next)
    except:
        next

    try:
        tcp_flags += ("URG" if tcp_flags_str.find('U') != -1 else next)
    except:
        next

    try:
        tcp_flags += ("ECE" if tcp_flags_str.find('E') != -1 else next)
    except:
        next

    try:
        tcp_flags += ("CWR" if tcp_flags_str.find('C') != -1 else next)
    except:
        next

    return tcp_flags

def analyse(df):
    """
    Analysis only top traffic stream

    :param df:
    :return:
    """
    debug = True
    attack_case = "-1"
    ttl_variation_threshold = 4

    result = {
        "reflected":False,
        "spoofed":False,
        "fragmented":False,
        "pattern_traffic_share":0.0,
        "pattern_packet_count":0,
        "pattern_total_megabytes":0,
        "start_timestamp":0,
        "end_timestamp":0,
        "dst_ports":[], #(port,share)
        "src_ports":[], #(port,share)
        "ttl_variation":[],
        "src_ips":[],
        "dst_ips":[],
        "packets":[]
    }

    if debug: print("\n\n\n")
    top_ip_dst = df['ip_dst'].value_counts().index[0]
    if debug: print("Top dst IP: "+ top_ip_dst)
    result["dst_ips"] = top_ip_dst
    top_ip_proto = df[df['ip_dst'] == top_ip_dst]['ip_proto'].value_counts().index[0]
    if debug: print("Top IP protocol: "+str(top_ip_proto))

    ####
    # Performing a first filter based on the top_ip_dst (target IP), the source IPs canNOT be from the \16 of the
    # target IP, and the top IP protocol that targeted the top_ip_dst
    df_filtered = df[
        (df['ip_dst'] == top_ip_dst) & ~df['ip_src'].str.contains(".".join(top_ip_dst.split('.')[0:2]), na=False) & (
        df['ip_proto'] == top_ip_proto)]

    ####
    # Calculating the number of packets after the first filter
    total_packets_filtered = len(df_filtered)
    if debug: print("Number of packets: "+str(total_packets_filtered))
    result["total_nr_packets"] = total_packets_filtered

    ####
    # For attacks in the IP protocol level
    attack_label = get_ip_proto_name(top_ip_proto) + "-based attack"
    result["transport_protocol"] = get_ip_proto_name(top_ip_proto)

    ####
    # For attacks based on TCP or UDP, which have source and destination ports
    if ((top_ip_proto == 6) or (top_ip_proto == 17)):

        if debug: print("\n####################\nREMAINING :\n####################")
        ####
        # Calculating the distribution of source ports based on the first filter
        percent_src_ports = df_filtered['sport'].value_counts().divide(float(total_packets_filtered) / 100)

        if debug: print("\nSource ports frequency")
        if debug: print(percent_src_ports.head())

        ####
        # Calculating the distribution of destination ports after the first filter
        percent_dst_ports = df_filtered['dport'].value_counts().divide(float(total_packets_filtered) / 100)
        if debug: print("\nDestination ports frequency")
        if debug: print(percent_dst_ports.head())

        #####
        ## WARNING packets are filtered here again#####
        # Using the top 1 (source or destination) port to analyse a pattern of packets
        if (len(percent_src_ports) > 0) and (len(percent_dst_ports) > 0):
            if percent_src_ports.values[0] > percent_dst_ports.values[0]:
                if debug: print('Using top source port: ', percent_src_ports.keys()[0])
                df_pattern = df_filtered[df_filtered['sport'] == percent_src_ports.keys()[0]]
                result["selected_port"] = "src_" + str(percent_src_ports.keys()[0])
            else:
                if debug: print('Using top dest port: ', percent_dst_ports.keys()[0])
                df_pattern = df_filtered[df_filtered['dport'] == percent_dst_ports.keys()[0]]
                result["selected_port"] = "dst_" + str(percent_dst_ports.keys()[0])
        else:
            if debug: print('no top source/dest port')
            return None

        if debug: print("\n####################\nPATTERN "+ "\n####################")

        #####
        # Calculating the total number of packets involved in the attack
        pattern_packets = len(df_pattern)
        result["pattern_packet_count"] = pattern_packets

        #WARNING Can be wrong
        result['raw_attack_size_megabytes'] = (df_pattern['raw_size'].sum() /1000000).item()
        result["pattern_total_megabytes"] = (df_pattern[df_pattern['fragments'] == 0]['ip_length'].sum() / 1000000).item()

        #####
        # Calculating the percentage of the current pattern compared to the raw input file
        representativeness = float(pattern_packets) * 100 / float(total_packets_filtered)
        result["pattern_traffic_share"] = representativeness
        attack_label = 'In %.2f' % representativeness + "\n " + attack_label

        #####
        # Checking the existence of HTTP data
        http_data = df_pattern['http_data'].value_counts().divide(float(pattern_packets) / 100)

        #####
        # Checking the existence of TCP flags
        percent_tcp_flags = df_pattern['tcp_flag'].value_counts().divide(float(pattern_packets) / 100)

        #####
        # Calculating the number of source IPs involved in the attack
        ips_involved = df_pattern['ip_src'].unique()
        attack_label = attack_label + "\n"+ str(len(ips_involved)) + " source IPs"
        result["src_ips"] = ips_involved.tolist()

        #####
        # Calculating the number of source IPs involved in the attack
        result["start_timestamp"] = df_pattern['timestamp'].min().item()
        result["end_timestamp"] = df_pattern['timestamp'].max().item()

        ####
        # Calculating the distribution of TTL variation (variation -> number of IPs)
        ttl_variations = df_pattern.groupby(['ip_src'])['ip_ttl'].agg(np.ptp).value_counts().sort_index()
        if debug: print('TTL variation : NR of source IPs')
        if debug: print(ttl_variations)
        #use to_json
        result["ttl_variation"] = json.loads(ttl_variations.to_json())

        ####
        # Calculating the distribution of IP fragments (fragmented -> percentage of packets)
        percent_fragments = df_pattern['fragments'].value_counts().divide(float(pattern_packets) / 100)

        ####
        # Calculating the distribution of source ports that remains
        percent_src_ports = df_pattern['sport'].value_counts().divide(float(pattern_packets) / 100)
        if debug: print("\nSource ports frequency")
        if debug: print(percent_src_ports.head())
        #use to_json()
        result["src_ports"] = json.loads(percent_src_ports.to_json())

        ####
        # Calculating the distribution of destination ports after the first filter
        percent_dst_ports = df_pattern['dport'].value_counts().divide(float(pattern_packets) / 100)
        if debug: print("\nDestination ports frequency")
        if debug: print(percent_dst_ports.head())
        #use to_json
        result["dst_ports"] = json.loads(percent_dst_ports.to_json())

        ####
        # There are 3 possibilities of attacks cases!
        if (percent_src_ports.values[0] == 100):
            if (len(percent_dst_ports) == 1):
                # if debug: print("\nCASE 1: 1 source port to 1 destination port") if debug else next
                attack_label = attack_label + "; using " + get_port_name(
                    percent_src_ports.keys()[0]) + "; to target " + get_port_name(
                    percent_dst_ports.keys()[0]) + "[" + '%.1f' % percent_dst_ports.values[0] + "%]"
            else:
                # if debug: print("\nCASE 2: 1 source port to a set of destination ports") if debug else next
                if (percent_dst_ports.values[0] >= 50):
                    attack_label = attack_label + "; using " + get_port_name(
                        percent_src_ports.keys()[0]) + "; to target a set of (" + str(
                        len(percent_dst_ports)) + ") ports, such as " + get_port_name(
                        percent_dst_ports.keys()[0]) + "[" + '%.2f' % percent_dst_ports.values[
                        0] + "%]" + " and " + get_port_name(percent_dst_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_dst_ports.values[
                                                                                1] + "%]"
                elif (percent_dst_ports.values[0] >= 33):
                    attack_label = attack_label + "; using " + get_port_name(
                        percent_src_ports.keys()[0]) + "; to target a set of (" + str(
                        len(percent_dst_ports)) + ") ports, such as " + get_port_name(
                        percent_dst_ports.keys()[0]) + "[" + '%.2f' % percent_dst_ports.values[
                        0] + "%]" + "; " + get_port_name(percent_dst_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_dst_ports.values[
                                                                                1] + "%], and " + get_port_name(
                        percent_dst_ports.keys()[2]) + "[" + '%.2f' % percent_dst_ports.values[2] + "%]"
                else:
                    attack_label = attack_label + "; using " + get_port_name(
                        percent_src_ports.keys()[0]) + "; to target a set of (" + str(
                        len(percent_dst_ports)) + ") ports, such as " + get_port_name(
                        percent_dst_ports.keys()[0]) + "[" + '%.2f' % percent_dst_ports.values[
                        0] + "%]" + "; " + get_port_name(percent_dst_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_dst_ports.values[
                                                                                1] + "%], and " + get_port_name(
                        percent_dst_ports.keys()[2]) + "[" + '%.2f' % percent_dst_ports.values[2] + "%]"
        else:
            if (len(percent_src_ports) == 1):
                # if debug: print("\nCASE 1: 1 source port to 1 destination port") if debug else next
                attack_label = attack_label + "; using " + get_port_name(percent_src_ports.keys()[0]) + "[" + '%.1f' % \
                                                                                percent_src_ports.values[
                                                                                0] + "%]" + "; to target " + get_port_name(
                    percent_dst_ports.keys()[0]) + "[" + '%.1f' % percent_dst_ports.values[0] + "%]"

            else:
                # if debug: print("\nCASE 3: 1 source port to a set of destination ports") if debug else next
                if (percent_src_ports.values[0] >= 50):
                    attack_label = attack_label + "; using a set of (" + str(
                        len(percent_src_ports)) + ") ports, such as " + get_port_name(
                        percent_src_ports.keys()[0]) + "[" + '%.2f' % percent_src_ports.values[
                        0] + "%] and " + get_port_name(percent_src_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_src_ports.values[
                                                                                1] + "%]" + "; to target " + get_port_name(
                        percent_dst_ports.keys()[0]) + "[" + '%.1f' % percent_dst_ports.values[0] + "%]"
                elif (percent_src_ports.values[0] >= 33):
                    attack_label = attack_label + "; using a set of (" + str(
                        len(percent_src_ports)) + ") ports, such as " + get_port_name(
                        percent_src_ports.keys()[0]) + "[" + '%.2f' % percent_src_ports.values[
                        0] + "%], " + get_port_name(percent_src_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_src_ports.values[
                                                                                1] + "%], and " + get_port_name(
                        percent_src_ports.keys()[2]) + "[" + '%.2f' % percent_src_ports.values[
                        2] + "%]" + "; to target " + get_port_name(percent_dst_ports.keys()[0]) + "[" + '%.1f' % \
                                                                                percent_dst_ports.values[
                                                                                0] + "%]"
                else:
                    attack_label = attack_label + "; using a set of (" + str(
                        len(percent_src_ports)) + ") ports, such as " + get_port_name(
                        percent_src_ports.keys()[0]) + "[" + '%.2f' % percent_src_ports.values[
                        0] + "%], " + get_port_name(percent_src_ports.keys()[1]) + "[" + '%.2f' % \
                                                                                percent_src_ports.values[
                                                                                1] + "%], " + get_port_name(
                        percent_src_ports.keys()[2]) + "[" + '%.2f' % percent_src_ports.values[
                        2] + "%]" + "; and " + get_port_name(percent_src_ports.keys()[3]) + "[" + '%.2f' % \
                                                                                percent_src_ports.values[
                                                                                3] + "%]" + "; to target " + get_port_name(
                        percent_dst_ports.keys()[0]) + "[" + '%.1f' % percent_dst_ports.values[0] + "%]"

        ####
        # Testing HTTP request
        if len(http_data) > 0 and ((percent_dst_ports.index[0] == 80) or (percent_dst_ports.index[0] == 443)):
            attack_label = attack_label + "; " + http_data.index[0]

        ####
        # Testing TCP flags
        if (len(percent_tcp_flags) > 0) and (percent_tcp_flags.values[0] > 50):
            attack_label = attack_label + "; TCP flags: " + get_tcp_flag_name(
                percent_tcp_flags.index[0]) + "[" + '%.1f' % percent_tcp_flags.values[0] + "%]"

        ####
        # IP fragmentation
        if (percent_fragments.values[0] > 0) and (percent_fragments.index[0] == 1):
            attack_label = attack_label + "; involving IP fragmentation"
            result["fragmented"] = True

        ####
        # IP spoofing (if (more than 0) src IPs had the variation of the ttl higher than a treshold)
        if (ttl_variations[::-1].values[0] > 0) and (ttl_variations[::-1].index[0] >= ttl_variation_threshold):
            result["spoofed"]=True
            attack_label = attack_label + "; (likely involving) spoofed IPs"
        else:
            ####
            # Reflection and Amplification
            if percent_src_ports.values[0] >= 1:
                result["reflected"]=True
                attack_label = attack_label + "; Reflection & Amplification"

        if debug: print("\n####################\nATTACK VECTOR LABEL:"+ "\n####################")
        if debug: print(attack_label)

        result["label"] = attack_label
        print(result)

    return result

if __name__ == "__main__":
    cli.add_command(receive)
    cli()


import scrapy
from module.items import HotlineItem

class HotlineSpider(scrapy.Spider):
    name = "hotline"
    allowed_domains = ["hotline.ua"]
    start_urls = [f"https://hotline.ua/ua/bt/holodilniki/?p={page}" for page in range(1, 4)]

    def parse(self, response):
        catalog = response.xpath('//div[contains(@class, "list-body__content")]').xpath('//div[contains(@class, "list-item ")]')

        for item in catalog:
            url = item.xpath('.//a[contains(@class, "list-item__title")]/@href').get()

            yield scrapy.Request(
                url=f"https://hotline.ua" + url,
                callback=self.parse_hotline
            )

    def parse_hotline(self, response):
        product_name = response.xpath('.//h1[contains(@class, "title__main")]/text()').get()
        store_name = response.xpath('.//div[1][@class="list__item row flex"]//a[@data-eventcategory="Pages Product Prices" and @class="shop__title"]/text()').get()
        price = response.xpath('.//div[1][@class="list__item row flex"]//span[@data-eventcategory="Pages Product Prices"]/span[@class="price__value"]/text()').get()

        yield HotlineItem(
            store_name=store_name,
            product_name=product_name,
            price=price
        )


import os
import requests
import json
from requests_oauthlib import OAuth1
from dotenv import load_dotenv

load_dotenv()

CONSUMER_KEY = os.getenv("TWITTER_API_KEY")
CONSUMER_SECRET = os.getenv("TWITTER_API_SECRET")
ACCESS_TOKEN = os.getenv("TWITTER_ACCESS_TOKEN")
ACCESS_TOKEN_SECRET = os.getenv("TWITTER_ACCESS_TOKEN_SECRET")
API_KEY = os.getenv("WAKATIME_API_KEY")

auth = OAuth1(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

def get_wakatime_stats():
    waka_url = (
        f"https://wakatime.com/api/v1/users/current/status_bar/today?api_key={API_KEY}"
    )
    response = requests.get(waka_url)
    if response.status_code != 200:
        raise Exception(
            f"Request failed with error {response.status_code}, {response.text}"
        )

    stats = {}
    top_3_languages = {}
    stats["minutes_coded_today"] = response.json()["data"]["grand_total"]["text"]

    for language in response.json()["data"]["languages"][0:3]:
        top_3_languages[language["name"]] = language["text"]
    return stats, top_3_languages

def update_coding_streak():
    with open("coding_streak.txt", "r") as f:
        coding_streak = int(f.read().strip())

    stats, top_3_languages = get_wakatime_stats()
    if stats["minutes_coded_today"]:
        coding_streak += 1
    else:
        coding_streak = 0

    with open("coding_streak.txt", "w") as f:
        f.write(str(coding_streak))

    # If the coding streak is broken, do not tweet and end the program.  The streak file will be updated to 0.
    if coding_streak == 0:
        print("Coding streak broken, not tweeting")
        exit()

    return coding_streak

def format_tweet(coding_streak, minutes_coded_today, languages):
    day_of_streak = coding_streak
    tweet_text = f"Day {day_of_streak} of my coding streak: I coded {minutes_coded_today} today!\n\n#100DaysOfCode "
    # make the hashtags each language
    for language in languages.keys():
        tweet_text += f"#{language} "

    return tweet_text

def post_tweet(tweet_text):
    url = "https://api.twitter.com/2/tweets"
    headers = {"Content-Type": "application/json"}
    data = {"text": tweet_text}
    response = requests.post(url, auth=auth, data=json.dumps(data), headers=headers)

    if response.status_code != 201:
        raise Exception(
            f"Request failed with error {response.status_code}, {response.text}"
        )

    print("Tweet posted successfully!", response.json())

if __name__ == "__main__":
    coding_streak = update_coding_streak()
    stats, languages = get_wakatime_stats()
    tweet_text = format_tweet(coding_streak, stats["minutes_coded_today"], languages)
    post_tweet(tweet_text)


"""fask_dj URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^tasks', views.tasks, name = 'tasks'),
    url(r'^task/(?P<task_id>[0-9]+)/$', views.task, name = 'task'),
    url(r'^task_delete/(?P<task_id>[0-9]+)/$', views.task_delete, name = 'task_delete'),
    url(r'^task_project/(?P<task_id>[0-9]+)/(?P<project_id>[0-9]+)/(?P<redirect_name>[a-z]+)$', views.task_project, name = 'task_project'),
]


from django.db import models
import uuid

class apiKeys(models.Model):
    id = models.CharField(max_length=200, null=False, blank=False)
    key = models.TextField(primary_key=True, default=uuid.uuid4().hex)

    def __unicode__(self):
        return self.id

class ButtonTable(models.Model):
    SKRANKE = 0
    TELEFON = 1
    RANDOM = 2
    BUTTON_CHOICES= (
        (SKRANKE, 'Skranke'),
        (TELEFON, 'Telefon'),
        (RANDOM, 'Random')
    )
    sender = models.ForeignKey(apiKeys, editable=False, null=False)
    button = models.IntegerField(choices=BUTTON_CHOICES, default=0)
    date_registered = models.DateTimeField(auto_now_add=True, auto_now=False)

    def __unicode__(self):
        return str(self.button)

    class Meta:
        verbose_name_plural = 'Case registers'
        verbose_name = 'Registered case'


# coding: utf-8
"""
Django settings for testproject project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import sys

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

sys.path.insert(0, os.path.dirname(BASE_DIR))

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'b2k427m)td!v8h-0@=qd-9pvvu@gd_zl-0^i@6j589uhp=egsx'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'modeltranslation',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'testproject',
    'haystack',
    'parler',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'testproject.urls'

WSGI_APPLICATION = 'testproject.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'myapp.sqlite',
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en'
LANGUAGES = (('en', 'English'),
             ('de', 'Deutsch'),
             ('es', 'Spanisch'),
             ('fr', 'Französisch'),
             ('cs', 'Tschechisch'),
             ('ru', 'Russisch'),
             )

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_URL = '/static/'

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'multilingual.elasticsearch_backend.ElasticsearchMultilingualSearchEngine',
        'URL': 'http://127.0.0.1:9200/',
        'INDEX_NAME': 'testproject',
        'SILENTLY_FAIL': False,
        'TIMEOUT': 30,
    },
}

HAYSTACK_LOGGING = True

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
        },
        'haystack': {
            'handlers': ['console'],
            'level': 'DEBUG'
        }
    },
}

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
    },
]


from web3 import Web3, EthereumTesterProvider
from etherscan import Etherscan
import json

class Node:
    def __init__(self, name, value, address, children=[]):
        self.name = name
        self.value = value
        self.children = children
        self.searched = False
        self.address = Web3.toChecksumAddress(address)

class NodeOperations:
    def __init__(self, web3, etherscan):
        self.web3 = web3
        self.etherscan = etherscan

    def print_tree(self, node, level=0):
        print('    ' * level + node.name + ": \t" + node.address)
        for child in node.children:
            self.print_tree(child, level+1)

    def depth_first_search(self, node, depth, current_depth=0, nodeIterator=0):
        if current_depth >= depth: #Escape after depth reached or exceeded
            return
        if node.searched == True:
            return

        availibleABI = True

        #Handler for non verified smart contracts on etherscan
        try:
            contractABI = self.etherscan.get_contract_abi(node.address) #Use Etherscan API to get the contracts ABI
        except:
            print('Problem with the ABI')
            availibleABI = False #unavailible ABI (contract code not verified)

        if availibleABI: #If contract's ABI is verified
            contractInstance = self.web3.eth.contract(address=node.address, abi=contractABI) #Use web3 Library to create an instantiation of the contract
            contractABI = json.loads(contractABI) #convert ABI to json format

            for i in range(len(contractABI)): #Examine all functions/methods/variables in the ABI
                try:
                    if contractABI[i]['outputs'][0]['type'] == 'address': #Searching exclusively for addresses on the contract
                        if len(contractABI[i]['inputs']) == 0:  #Check function call does not require input
                            childAddress = eval("contractInstance."+"functions."+contractABI[i]['name']+"()"+".call()") #RPC call to the contract, return the 20byte address
                            child = Node(contractABI[i]['name'], nodeIterator, childAddress, []) #create node
                            node.children.append(child) #Append child node
                            #nodeIterator += 1
                            #print(f'Searching node...')
                        elif len(contractABI[i]['inputs']) == 1 and contractABI[i]['inputs'][0]['type'] == 'uint256': #This is an array of addresses:
                            print('Searching Array Node...')
                            try:
                                for j in range(0,10):
                                    #print(j)
                                    childAddress = eval("contractInstance."+"functions."+contractABI[i]['name']+"("+str(j)+ ")"+".call()") #RPC call to the contract, return the 20byte address
                                    child = Node(contractABI[i]['name'], nodeIterator, childAddress, []) #create node
                                    node.children.append(child) #Append child node
                                    nodeIterator += 1
                                    #print(f'Searching node...') #NOTE: Might need to put a check in for 0x0000... etc address.
                            except:
                                break
                        else:
                            pass
                except:
                    pass

            for child in node.children:
                child.name
                self.depth_first_search(child, depth, current_depth+1, nodeIterator)

            node.searched = True
        else:
            print(node.address)
            node.name = 'NO ABI AVAILIBLE'
            node.searched = True
            pass


from Epulet import Epulet

epuletek = []

def beolvas(fajlnev):

    fajl = open(fajlnev, "r", encoding="utf-8")
    fajl.readline()
    sorok = fajl.readlines()
    fajl.close()

    i = 0
    while i < len(sorok):
        sor = sorok[i].strip().split("$")
        epulet = Epulet(sor)
        epuletek.append(epulet)
        i += 1

def epuletek_szama():
    return len(epuletek)

def magasabbmint():
    magasabb = 0
    i = 0
    while i < len(epuletek):

        if epuletek[i].magassag * 3.280839895 > 555:
            magasabb += 1
        i += 1

    return magasabb

def legoregebb_orszag():
    legoregebb = 99999
    oregorszag = "hiba"

    i = 0
    while i < len(epuletek):
        if epuletek[i].ev < legoregebb:
            legoregebb = epuletek[i].ev
            oregorszag = epuletek[i].orszag
        i += 1

    return oregorszag


# -*- coding: utf-8 -*-
from enum import Enum

class SolarTerms(Enum):
    the_beginning_of_spring = "the Beginning of Spring", "立春"
    rain_water = "Rain Water", "雨水"
    the_waking_of_insects = "the Waking of Insects", "惊蛰"
    the_spring_equinox = "the Spring Equinox", "春分"
    pure_brightness = "Pure Brightness", "清明"
    grain_rain = "Grain Rain", "谷雨"
    the_beginning_of_summer = "the Beginning of Summer", "立夏"
    lesser_fullness_of_grain = "Lesser Fullness of Grain", "小满"
    grain_in_beard = "Grain in Beard", "芒种"
    the_summer_solstice = "the Summer Solstice", "夏至"
    lesser_heat = "Lesser Heat", "小暑"
    greater_heat = "Greater Heat", "大暑"
    the_beginning_of_autumn = "the Beginning of Autumn", "立秋"
    the_end_of_heat = "the End of Heat", "处暑"
    white_dew = "White Dew", "白露"
    the_autumn_equinox = "the Autumn Equinox", "秋分"
    code_dew = "Cold Dew", "寒露"
    frost_descent = "Frost's Descent", "霜降"
    the_beginning_of_winter = "the Beginning of Winter", "立冬"
    lesser_snow = "Lesser Snow", "小雪"
    greater_snow = "Greater Snow", "大雪"
    the_winter_solstice = "the Winter Solstice", "冬至"
    lesser_cold = "Lesser Cold", "小寒"
    greater_cold = "Greater Cold", "大寒"

# 计算节气用的 C 值
# 2000年的小寒、大寒、立春、雨水按照20世纪的C值来算
SOLAR_TERMS_C_NUMS = {
    # 节气: [20世纪值， 21世纪值]
    SolarTerms.the_beginning_of_spring: [4.6295, 3.87],
    SolarTerms.rain_water: [19.4599, 18.73],
    SolarTerms.the_waking_of_insects: [6.3926, 5.63],
    SolarTerms.the_spring_equinox: [21.4155, 20.646],
    SolarTerms.pure_brightness: [5.59, 4.81],
    SolarTerms.grain_rain: [20.888, 20.1],
    SolarTerms.the_beginning_of_summer: [6.318, 5.52],
    SolarTerms.lesser_fullness_of_grain: [21.86, 21.04],
    SolarTerms.grain_in_beard: [6.5, 5.678],
    SolarTerms.the_summer_solstice: [22.2, 21.37],
    SolarTerms.lesser_heat: [7.928, 7.108],
    SolarTerms.greater_heat: [23.65, 22.83],
    SolarTerms.the_beginning_of_autumn: [28.35, 7.5],
    SolarTerms.the_end_of_heat: [23.95, 23.13],
    SolarTerms.white_dew: [8.44, 7.646],
    SolarTerms.the_autumn_equinox: [23.822, 23.042],
    SolarTerms.code_dew: [9.098, 8.318],
    SolarTerms.frost_descent: [24.218, 23.438],
    SolarTerms.the_beginning_of_winter: [8.218, 7.438],
    SolarTerms.lesser_snow: [23.08, 22.36],
    SolarTerms.greater_snow: [7.9, 7.18],
    SolarTerms.the_winter_solstice: [22.6, 21.94],
    SolarTerms.lesser_cold: [6.11, 5.4055],
    SolarTerms.greater_cold: [20.84, 20.12],
}

# 月份和节气对应关系
SOLAR_TERMS_MONTH = {
    1: [SolarTerms.lesser_cold, SolarTerms.greater_cold],
    2: [SolarTerms.the_beginning_of_spring, SolarTerms.rain_water],
    3: [SolarTerms.the_waking_of_insects, SolarTerms.the_spring_equinox],
    4: [SolarTerms.pure_brightness, SolarTerms.grain_rain],
    5: [SolarTerms.the_beginning_of_summer, SolarTerms.lesser_fullness_of_grain],
    6: [SolarTerms.grain_in_beard, SolarTerms.the_summer_solstice],
    7: [SolarTerms.lesser_heat, SolarTerms.greater_heat],
    8: [SolarTerms.the_beginning_of_autumn, SolarTerms.the_end_of_heat],
    9: [SolarTerms.white_dew, SolarTerms.the_autumn_equinox],
    10: [SolarTerms.code_dew, SolarTerms.frost_descent],
    11: [SolarTerms.the_beginning_of_winter, SolarTerms.lesser_snow],
    12: [SolarTerms.greater_snow, SolarTerms.the_winter_solstice],
}

# 有些节气使用公式计算计算的不够准确，需要进行偏移
SOLAR_TERMS_DELTA = {
    (2026, SolarTerms.rain_water): -1,
    (2084, SolarTerms.the_spring_equinox): 1,
    (1911, SolarTerms.the_beginning_of_summer): 1,
    (2008, SolarTerms.lesser_fullness_of_grain): 1,
    (1902, SolarTerms.grain_in_beard): 1,
    (1928, SolarTerms.the_summer_solstice): 1,
    (1925, SolarTerms.lesser_heat): 1,
    (2016, SolarTerms.lesser_heat): 1,
    (1922, SolarTerms.greater_heat): 1,
    (2002, SolarTerms.the_beginning_of_autumn): 1,
    (1927, SolarTerms.white_dew): 1,
    (1942, SolarTerms.the_autumn_equinox): 1,
    (2089, SolarTerms.frost_descent): 1,
    (2089, SolarTerms.the_beginning_of_winter): 1,
    (1978, SolarTerms.lesser_snow): 1,
    (1954, SolarTerms.greater_snow): 1,
    (1918, SolarTerms.the_winter_solstice): -1,
    (2021, SolarTerms.the_winter_solstice): -1,
    (1982, SolarTerms.lesser_cold): 1,
    (2019, SolarTerms.lesser_cold): -1,
    (2000, SolarTerms.greater_cold): 1,
    (2082, SolarTerms.greater_cold): 1,
}


from django.shortcuts import render, get_object_or_404
from .models import Post, Comment
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from .forms import CommentForm
from django.views.decorators.http import require_POST
from taggit.models import Tag
"""
This is the Count aggregation function of the Django ORM. This function will
allow you to perform aggregated counts of tags. django.db.models includes the
following aggregation functions:
• Avg: The mean value
• Max: The maximum value
• Min: The minimum value
• Count: The total number of objects
"""
from django.db.models import Count
from itertools import chain

# Define the number of elements in one page
page_elements_count = 6

# Define number of similar posts
similar_posts_count = 3

# Create your views here.
def blog_list(request, tag_slug=None):

    post_list = Post.published.all()

    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        post_list = post_list.filter(tags__in=[tag])

    posts_count = post_list.count()

    # Pagination with page_elements_count per page
    paginator = Paginator(post_list, page_elements_count)

    # f the page parameter is not in the GET parameters of the request, we use
    # the default value 1 to load the first page of results.
    page_number = request.GET.get('page', 1)

    # Try get the requested page
    try:
        posts = paginator.page(page_number)

    # If the requests page is out of range, retrun the last page
    except EmptyPage:
        # If page_number is out of range deliver last page of results
        posts = paginator.page(paginator.num_pages)

    # If the requests page is not an integer, return the first page
    except PageNotAnInteger:
        posts = paginator.page(1)

    return render(
        request,
        'blog/blog_home.html',
        {
            'posts': posts,
            'posts_count': posts_count,
            'tag': tag,
        }
    )

def blog_post(request, post_slug):

    post = get_object_or_404(
        Post,
        slug=post_slug,
        status=Post.Status.PUBLISHED
    )
    # List of active comments for this post
    comments = post.comments.filter(active=True)

    if request.method == 'GET':

        # Form for users to comment
        form = CommentForm()

    else:

        # Define a comment variable with the initial value None. This variable will
        # be used to store the comment object when it gets created.
        comment = None

        form = CommentForm(data=request.POST)

        if form.is_valid():
            comment = form.save(commit=False)

            # Assign the post attribute of the comment to the current post
            comment.post = post

            # Save the comment to the database
            comment.save()

    # List of similar posts

    # retrieve a Python list of IDs for the tags of the current post. The
    # values_list() QuerySet returns tuples with the values for the given
    # fields. We pass flat=True to it to get single values such as
    # [1, 2, 3, ...] instead of one-tuples such as [(1,), (2,), (3,) ...]
    post_tags_ids = post.tags.values_list('id', flat=True)

    # We get all posts that contain any of these tags, excluding the current
    # post itself.
    similar_posts = Post.published.filter(tags__in=post_tags_ids).exclude(id=post.id)

    # use the Count aggregation function to generate a calculated
    # field (same_tags) that contains the number of tags shared with all the
    # tags queried.
    # We order the result by the number of shared tags (descending order) and
    # by publish to display recent posts first for the posts with the same
    # number of shared tags. We slice the result to retrieve only the first
    # three posts.
    similar_posts = similar_posts.annotate(same_tags=Count('tags'))\
                        .order_by('-same_tags', '-created')[:similar_posts_count]

    # Fill similar posts with normal posts; if the similar posts are less than
    # the required similar_posts_count
    if len(similar_posts) < similar_posts_count:
        extra_posts_count = similar_posts_count - len(similar_posts)
        extra_posts = Post.published.order_by('-created').exclude(id=post.id)\
                          .exclude(id__in=similar_posts)[:extra_posts_count]
        similar_posts = list(chain(similar_posts, extra_posts))

    return render(
        request,
        'blog/blog_post.html',
        {
            'post': post,
            'form': form,
            'comments': comments,
            'similar_posts': similar_posts,
        }
    )


# Creating a regression line - aka line of best fit
# Y = mx + b
from statistics import mean
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style

style.use('fivethirtyeight')

# defined as numpy arrays
xs = np.array([1,2,3,4,5,6], dtype=np.float64)
ys = np.array([5,4,6,5,6,7], dtype=np.float64)

def best_fit_slope_and_intercept(xs,ys):
    m = (((mean(xs) * mean(ys)) - mean(xs*ys)) / ( (mean(xs) * mean(xs)) - mean(xs**2)    ) )
    b = mean(ys) - m*mean(xs)
    return m,b;

m,b = best_fit_slope_and_intercept(xs,ys)

#Creating line via for loop
regression_line = [(m*x)+b for x in xs]

#print(m,b)

predict_x = 8
predict_y = (m*predict_x + b)

#Create plots
plt.scatter(xs,ys)
plt.scatter(predict_x,predict_y)
plt.plot(xs, regression_line)
plt.show()


with open("day3/input.txt") as f:
    s = 0
    for l in f:
        comp1, comp2 = l[:int(len(l)/2)], l[int(len(l)/2):]
        common = list(set(comp1).intersection(set(comp2)))
        val = ord(common[0]) - 38 if common[0].isupper() else ord(common[0]) - 96
        s += val
    print(s)

with open("day3/input.txt") as f:
    s = 0
    group = []
    for l in f:
        l = l.replace("\n", "")
        group.append(l)
        if len(group) == 3:
            common = list(set.intersection(set(group[0]), set(group[1]), set(group[2])))
            val = ord(common[0]) - 38 if common[0].isupper() else ord(common[0]) - 96
            s += val
            group = []

    print(s)


import numpy as np

#=========================#
# refine the distribution #
#=========================#
def refine_distribution(old_mugrid, old_mumid, old_dist, target_resolution):
    # store moments for comparison
    old_nmu = len(old_mumid)
    old_edens = np.sum(old_dist               , axis=(1,2,3))
    old_flux  = np.sum(old_dist * old_mumid   , axis=(1,2,3))
    old_press = np.sum(old_dist * old_mumid**2, axis=(1,2,3))

    # initially set to old values in case not refining
    new_mugrid = old_mugrid
    new_mumid = old_mumid
    new_dist = old_dist
    new_nmu = old_nmu

    nref =  int(np.log2(target_resolution / old_nmu))
    for iref in range(nref):
        if(iref>0):
            old_dist = new_dist
            old_mugrid = new_mugrid

        # get the new mugrid
        old_nmu = len(old_mugrid)-1
        old_dmu = np.array([old_mugrid[i+1]-old_mugrid[i] for i in range(old_nmu)])
        new_mugrid = []
        for i in range(len(old_mugrid)-1):
            new_mugrid.append(old_mugrid[i])
            new_mugrid.append(0.5*(old_mugrid[i]+old_mugrid[i+1]))
        new_mugrid.append(old_mugrid[-1])
        new_nmu = len(new_mugrid)-1
        new_dmu = np.array([new_mugrid[i+1]-new_mugrid[i] for i in range(new_nmu)]) # dist between cell faces
        print("\tIncreasing from nmu=",old_nmu,"to nmu=",new_nmu)

        # get properties & derivatives of the old distribution function
        nr = old_dist.shape[0]
        ns = old_dist.shape[1]
        ne = old_dist.shape[2]

        new_dist = np.zeros((nr,ns,ne,new_nmu))
        for i in range(old_nmu):
            new_dist[:,:,:,2*i+0] = old_dist[:,:,:,i]/2.
            new_dist[:,:,:,2*i+1] = old_dist[:,:,:,i]/2.

    # new grid structure #
    if nref > 0:
        new_mumid = np.array([0.5*(new_mugrid[i]+new_mugrid[i+1]) for i in range(new_nmu)])

    # calculate error in moments #
    new_edens = np.sum(new_dist               , axis=(1,2,3))
    new_flux  = np.sum(new_dist * new_mumid   , axis=(1,2,3))
    new_press = np.sum(new_dist * new_mumid**2, axis=(1,2,3))
    print("  DO Max relative error in net energy density:", np.max(np.abs((new_edens-old_edens)/old_edens)))
    print("  DO Max relative error in net flux:", np.max(np.abs((new_flux-old_flux)/old_edens)))
    print("  DO Max relative error in net pressure:", np.max(np.abs((new_press-old_press)/old_edens)))

    return new_mugrid, new_mumid, new_dist