# implementation based on DeepLTL https://github.com/reactive-systems/deepltl

import sys
import importlib
import os.path as path
import subprocess
import re
from timeit import default_timer as timer
from typing import Tuple

import multiexit

from dlsgs.utils import ltl_parser

DEFAULT_BINARY_PATH = "bin"

def solve_ltl(formula_obj, tool, worker, timeout=None, simplify=False, no_disjunctions=False, witness=True, binary_path=DEFAULT_BINARY_PATH):
    if tool == 'spot':
        formula_str = formula_obj.to_str('spot', spacing='all ops', full_parens=True) # unambiguous form
        finished, res = worker.call(_solve_spot, (formula_str, simplify), timeout)
        if not finished:
            return None, None, {}  # timeout
        sat, trace, d = res
        assert sat is not None
        if sat:
            trace = ltl_parser.ltl_trace(trace, 'spot')
            if no_disjunctions:
                trace.decide_disjunctions()
        else:
            trace = None
        return sat, trace, d
    elif tool == 'aalta':
        formula_str = formula_obj.rewrite(ltl_parser.Token.WEAKUNTIL).to_str('spot', spacing='all ops', full_parens=True) # unambiguous form
        assert not simplify
        if witness:
            sat, trace = aalta_sat_with_evidence(formula_str, timeout=timeout, binary_path=binary_path)
            return sat, ltl_parser.ltl_trace(trace, 'spot') if sat else None, {}
        else:
            try:
                sat = aalta_sat(formula_str, timeout=timeout)
            except RuntimeError:
                sat = None
            return sat, None, {}
    elif tool == 'leviathan':
        formula_str = formula_obj.to_str('spot', spacing='all ops', full_parens=True) # unambiguous form
        assert not simplify
        worker.register_close_function(_close_leviathan)
        finished, res = worker.call(_solve_leviathan, (formula_str, binary_path), timeout)
        if not finished:
            return None, None, {} # timeout
        sat, trace, d = res
        return sat, ltl_parser.ltl_trace(trace, 'spot', symbolic=False) if sat else None, d
    else:
        raise ValueError("Unknown tool")

def _solve_leviathan(formula_str, binary_path=DEFAULT_BINARY_PATH):
    if not 'proc' in globals():
        global proc
        lev_path = path.join(binary_path, 'leviathan')
        proc = subprocess.Popen([lev_path, '-m', '-p', '-v', '4', '/dev/stdin'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1)
        multiexit.register(proc.kill)

    t_start = timer()
    #print(formula_str)
    sys.stdout.flush()
    proc.stdin.write(formula_str + '\n')
    stuff = ''
    while True:
        lastline = proc.stdout.readline()
        if lastline.startswith('SAT;') or lastline.startswith('UNSAT'):
            break
        else:
            stuff += lastline
    d = {'solve_time' : (timer() - t_start) * 1000 }
    d.update(find_int_in_output(r'Found (\d+) eventualities\n', stuff, 'lev_evtls', None))
    d.update(find_int_in_output(r'Total frames: (\d+)\n', stuff, 'lev_frames', None))
    d.update(find_int_in_output(r'Total STEPs: (\d+)\n', stuff, 'lev_steps', None))
    d.update(find_int_in_output(r'Maximum model size: (\d+)\n', stuff, 'lev_max_model_size', None))
    d.update(find_int_in_output(r'Maximum depth: (\d+)\n', stuff, 'lev_max_depth', None))

    if lastline.startswith('SAT;'):
        sat = True
        lastline = lastline[4:]
        parts = lastline.strip().split(' -> ')
        if len(parts) == 1:
            assert parts[0] == '{}'
            trace = 'cycle{1}'
        else:
            loop_to = re.match(r'#(\d+)', parts[-1])
            assert loop_to
            loop_to = int(loop_to.groups()[0])
            parts_ = []
            for p in parts[:-1]:
                assert (p.startswith('{') and p.endswith('}'))
                p = p[1:-1]
                if p:
                    parts_.append(p)
                else:
                    parts_.append('1')
            prefix = '; '.join(parts_[:loop_to])
            cycle =  '{' + '; '.join(parts_[loop_to:]) + '}'
            trace = prefix + ('; ' if prefix else '') + 'cycle' + cycle
    else: # unsat
        sat = False
        trace = None
    return sat, trace, d

def find_int_in_output(regex, output, name, default):
    m = re.search(regex, output)
    if m:
        return {name : int(m.groups()[0])}
    elif default is not None:
        return {name : default}
    else:
        return {}

def _close_leviathan():
    if 'proc' in globals():
        stdout, _ = proc.communicate(timeout=1)
        if stdout != '':
            print('Leviathan had trailing output:', stdout)
        proc.kill()

def _solve_spot(formula_str, simplify):
    if not 'spot' in globals():
        global spot
        spot = importlib.import_module('spot')
    t_start = timer()
    spot_formula = spot.formula(formula_str)
    automaton = spot_formula.translate()
    automaton.merge_edges()
    acc_run = automaton.accepting_run()
    d = {'solve_time' : (timer() - t_start) * 1000 }
    if acc_run is None:
        return False, None, d
    else:
        trace = spot.twa_word(acc_run)
        if simplify:
            trace.simplify()
        return True, str(trace), d

def aalta_sat_with_evidence(formula: str, timeout=None, binary_path=DEFAULT_BINARY_PATH) -> Tuple[bool, str]:
    """Calls aalta to check if the provided formula is satisfiable and returns a witness if so"""
    full_aalta_path = path.join(binary_path, 'aalta')
    try:
        # arguments -l and -c do not seem to be necessary, but include them, for future versions
        res = subprocess.run([full_aalta_path, '-l', '-c', '-e'], input=formula, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=True, universal_newlines=True)
    except subprocess.TimeoutExpired:
        print('aalta timed out after {:.2f}s'.format(timeout))
        return None, None
    except subprocess.CalledProcessError as e:
        raise RuntimeError("aalta threw an error: " + e.stderr)
    m = re.fullmatch('please input the formula:\n((?:un)?sat)\n(.*)', res.stdout, re.MULTILINE | re.DOTALL)
    if not m:
        raise RuntimeError("Regular expression for aalta output did not match. Output: '" + res.stdout + "'")
    res_sat, res_trace = m.groups()
    if res_sat == 'unsat':
        return False, None

    # convert aalta trace to spot trace
    assert res_sat == 'sat'
    m = re.fullmatch('(.*[(]\n.*\n[)])\\^w\n', res_trace, re.MULTILINE | re.DOTALL) # not really necessary, more as check
    if not m:
        raise RuntimeError("Regular expression for aalta trace did not match. Trace output: '" + res_trace + "'")
    trace_str = m.groups()[0]
    trace_str = re.sub('[{][}]\n', '1; ', trace_str)                                 # special case, yaay! convert {} directly to 1;
    trace_str = trace_str.replace('{', '').replace(',}', '').replace(',', ' & ')    # convert set {a, !b, c} to formula a & !b & c
    trace_str = trace_str.replace('true', '1')                                      # convert true literal to 1
    trace_str = re.sub('[(]\n', 'cycle{', trace_str)                                # convert ( ... ) to cycle{...}
    trace_str = re.sub('\n[)]$', '}', trace_str)
    trace_str = re.sub('\n', '; ', trace_str)                                       # convert newlines to ;
    return True, trace_str, {}

def aalta_sat(formula: str, timeout=None, binary_path=DEFAULT_BINARY_PATH, mute=False) -> bool:
    """Calls aalta to check if the provided formula is satisfiable"""
    full_aalta_path = path.join(binary_path, 'aalta')
    try:
        res = subprocess.run([full_aalta_path], input=formula, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=True, universal_newlines=True)
    except subprocess.TimeoutExpired:
        # print('aalta timed out after {:.2f}s'.format(timeout))
        return None
    except subprocess.CalledProcessError as e:
        if not mute:
            print("!! aalta threw an error: " + e.stderr)
            print('for input formula', formula)
        return None
    m = re.fullmatch('please input the formula:\n((?:un)?sat)\n', res.stdout, re.MULTILINE | re.DOTALL)
    if not m:
        raise RuntimeError("Regular expression for aalta output did not match. Output: '" + res.stdout + "'")
    res_sat = m.groups()[0]
    assert res_sat == 'sat' or res_sat == 'unsat'
    return res_sat == 'sat'


from unittest.mock import patch, Mock

import pytest

from item_db import ItemDB
from shopping_cart import ShoppingCart

@pytest.fixture
def cart():
    return ShoppingCart(4)

def mock_get_price(item: str):
    if item == 'pear':
        return 2
    if item == 'banana':
        return 3

def test_can_add_item_to_card(cart):
    cart.add('pear')

    assert cart.size() == 1

def test_when_item_added_then_cart_contains_item(cart):
    cart.add('raspberry')

    assert 'raspberry' in cart.get_items()

def test_when_more_than_max_size_should_fail(cart):
    [cart.add('pear') for x in range(4)]
    with pytest.raises(OverflowError):
        cart.add('pear')

def test_can_get_total_price(cart):
    cart.add('pear')
    cart.add('banana')

    price_map = {
        'pear': 2,
        'banana': 3
    }

    assert cart.get_total_price(price_map=price_map) == 5

def test_can_get_price_using_db(cart):
    cart.add('pear')
    cart.add('banana')

    item_db = ItemDB()
    item_db.get = Mock(side_effect=mock_get_price)

    assert cart.get_total_price(price_map=item_db) == 5


import glob
import os
import sys

if not os.geteuid() == 0:
    sys.exit('You must be root to run this application, please use sudo and try again.')

# fix input def in python2
try:
   input = raw_input
except NameError:
   pass

PATHS = ['.'];

for path in PATHS:
    path = os.path.abspath(path)
    projects = glob.glob(path + '/*/docker-compose.yml')
    if projects:
        print('--- ' + os.path.basename(path) + ' ---')
        for project in projects:
            print(os.path.basename(os.path.dirname(project)))

project = input('>>> ')

for path in PATHS:
    compose = glob.glob(path + '/' + project + '/docker-compose.yml')
    if compose:
        projectDir = os.path.dirname(compose[0])
        os.system('cd "' + projectDir + '" && docker-compose up -d')
    else:
        print('Project not found.')


import plotly.express as px
import pandas as pd
from datastructure import *
from sklearn.decomposition import PCA
import plotly.graph_objects as go

def createSPLOM(data, idx, indexRange, maxComp):
    O = np.transpose(data.series[idx].orientations, (1, 0, 2))[indexRange[0]:indexRange[1]]
    O = O.reshape(O.shape[0], data.particleNumber*3)#[-1000:]
    pca = PCA()
    pca_data = pca.fit_transform(O)
    var = pca.explained_variance_ratio_
    numDim = np.count_nonzero(var > 0.001)
    numDim = min(maxComp, numDim)
    pca_data = pca_data[:,:numDim]
    minVal = np.min(pca_data)-0.2
    maxVal = np.max(pca_data)+0.2
    df = pd.DataFrame(data=pca_data, index=[(""+str(i)) for i in range(len(pca_data))])
    splom = px.scatter_matrix(df,
                              labels={i:(""+str(i)) for i in range(numDim)})

    splom.update_traces(diagonal_visible=False)
    splom.update_layout(template = 'simple_white')
    splom.update_layout({"xaxis"+str(i+1): dict(range = [minVal, maxVal]) for i in range(numDim)})
    splom.update_layout({"yaxis"+str(i+1): dict(range = [minVal, maxVal]) for i in range(numDim)})
    splom.update_layout(font=dict(
        size=data.fontSize
    ))
    return splom


import argparse
import pprint
import shutil
from pathlib import Path

import tensorflow as tf
import yaml
from tensorflow import keras

from dataset_factory import DatasetBuilder
from losses import CTCLoss
from metrics import SequenceAccuracy
from models import build_model

parser = argparse.ArgumentParser()
parser.add_argument(
    "--config", type=Path, required=True, help="The config file path."
)
parser.add_argument(
    "--save_dir",
    type=Path,
    required=True,
    help="The path to save the models, logs, etc.",
)
args = parser.parse_args()

with args.config.open() as f:
    config = yaml.load(f, Loader=yaml.Loader)["train"]
pprint.pprint(config)

args.save_dir.mkdir(exist_ok=True)
if list(args.save_dir.iterdir()):
    raise ValueError(f"{args.save_dir} is not a empty folder")
shutil.copy(args.config, args.save_dir / args.config.name)

strategy = tf.distribute.MirroredStrategy()
batch_size = config["batch_size_per_replica"] * strategy.num_replicas_in_sync

dataset_builder = DatasetBuilder(**config["dataset_builder"])
train_ds = dataset_builder(config["train_ann_paths"], batch_size, True)
val_ds = dataset_builder(config["val_ann_paths"], batch_size, False)

with strategy.scope():
    lr_schedule = keras.optimizers.schedules.CosineDecay(
        **config["lr_schedule"]
    )
    model = build_model(
        dataset_builder.num_classes,
        weight=config.get("weight"),
        img_shape=config["dataset_builder"]["img_shape"],
    )
    model.compile(
        optimizer=keras.optimizers.Adam(lr_schedule),
        loss=CTCLoss(),
        metrics=[SequenceAccuracy()],
    )

model.summary()

model_prefix = "{epoch}_{val_loss:.4f}_{val_sequence_accuracy:.4f}"
model_path = f"{args.save_dir}/{model_prefix}.h5"
callbacks = [
    keras.callbacks.ModelCheckpoint(model_path, save_weights_only=True),
    keras.callbacks.TensorBoard(
        log_dir=f"{args.save_dir}/logs", **config["tensorboard"]
    ),
]

model.fit(
    train_ds,
    epochs=config["epochs"],
    callbacks=callbacks,
    validation_data=val_ds,
)


# This demo shows how to plot the contour plot for given function, with resultant of the partial derivatives displayed
# as arrows

import numpy as np
import matplotlib.pyplot as plt
import math

def func(x, y):
    z = y - x - 2 * x ** 2 - 2 * x * y - y ** 2
    return z

x = np.linspace(-2, 0, 20)
y = np.linspace(1, 3, 20)
[X, Y] = np.meshgrid(x, y)
Z = func(X, Y)
[fx, fy] = np.gradient(Z, 0.25)
vs1 = plt.contour(x, y, Z)
vs2 = plt.quiver(x, y, -fx, -fy)
plt.title("Contour plot & partial derivatives")
plt.show()


from heapq import *
from collections import deque
from collections import Counter

def schedule_tasks(tasks, k):

  # We should always try to schedule the task with the highest count.
  # So, we can get the counts for each task and add to max Heap.
  # Then, pop from maxHeap to schedule (k + 1) tasks
  # For each task, add to a waitlist
  # If we cannot add (k + 1) different tasks from max heap, we need
  # to increment interval count by the difference to insert idles
  # Then, put the tasks in the waitlist back into maxHeap

  intervalCount = 0
  maxHeap = []

  # Get Counts
  counts = Counter(tasks)

  # Put into maxHeap

  for char, count in counts.items():
    heappush(maxHeap, (-count, char))

  while maxHeap:
    waitList = []
    # Number of different tasks to try and execute
    n = k + 1

    # Try to Execute k + 1 tasks from maxHeap
    while maxHeap and n > 0:
      count, char = heappop(maxHeap)
      #print(char, end=", ")
      count += 1
      if count < 0:
        waitList.append((count, char))
      n -= 1
      intervalCount += 1
      #print(n)

    # Put waitlist tasks back into maxHeap
    for count, char in waitList:
      heappush(maxHeap, (count, char))

    # If we still have tasks to execute, but haven't done k + 1 tasks this iteration,
    # then we need to increment for idle tasks
    if maxHeap:
      intervalCount += n
      #for i in range(n):
        #print("idle", end=", ")
  #print("")

  return intervalCount

def main():
  print("Minimum intervals needed to execute all tasks: " +
        str(schedule_tasks(['a', 'a', 'a', 'b', 'c', 'c'], 2)))
  print("Minimum intervals needed to execute all tasks: " +
        str(schedule_tasks(['a', 'b', 'a'], 3)))
  print("Minimum intervals needed to execute all tasks: " +
        str(schedule_tasks(['a', 'a', 'b', 'b', 'c', 'c'], 2)))

main()


import os
import pathlib
import gzip
from itertools import groupby
from typing import List, Dict, TextIO, Tuple, Union

import numpy as np
import pandas as pd

import plot

# type aliases
NX_DICT = Dict[str, List[int]]
AUN_DICT = Dict[str, float]

class AssemblyCollection:

    def __init__(
        self,
        paths: List[Union[pathlib.Path, str]],
        reference_path: Union[pathlib.Path, str] = None,
        genome_size: int = None,
        out_name: str = 'pauNy'
    ):
        # make sure that input paths are pathlib.Paths
        self._assembly_paths = convert_paths(paths)
        self.ref_name = ''
        self.out_name = out_name

        if reference_path:
            # if reference file is given, save its name and load the genome size
            self.ref_path = pathlib.Path(reference_path)
            self.ref_name = self.ref_path.name
            self._assembly_paths.append(self.ref_path)
            self.genome_size = Reference(path=self.ref_path).genome_size()
        elif genome_size:
            # if genome size estimate is given
            self.genome_size = genome_size
        else:
            self.genome_size = 0
        # initialise an assembly object for all input files
        self.assemblies = self._initialise_assemblies(assembly_paths=self._assembly_paths)

    def calculate_metrics(self) -> None:
        """
        Calculate Nx and auN values for all assemblies in the collection

        :return: None
        """
        self.nx_values = self._calculate_Nx()
        self.aun_values = self._calculate_auN()
        # generate pandas frames and write to file
        self.nx_frame = self._generate_nx_frame(self.nx_values)
        self.aun_frame = self._generate_aun_frame(self.aun_values)
        # write to csv files
        print_frame(df=self.nx_frame, out_file=f'{self.out_name}.nx.csv')
        print_frame(df=self.aun_frame, out_file=f'{self.out_name}.aun.csv')

    def plot(self, format: str = 'pdf') -> None:
        """
        Generate a plot of Nx curves and auN values from all input assemblies

        :param format: Format passed to plotnine's save function
        :return: None
        """
        try:
            getattr(self, 'nx_frame')
        except AttributeError:
            self.calculate_metrics()

        plot.save_plot(
            plot=plot.plot_nx(self.nx_frame),
            type="nx",
            out_name=self.out_name,
            format=format
        )
        plot.save_plot(
            plot=plot.plot_aun(self.aun_frame),
            type="aun",
            out_name=self.out_name,
            format=format
        )

    def _calculate_Nx(self) -> NX_DICT:
        """
        Calculate Nx values for each assembly

        :return: Dict of Nx values per assembly
        """
        nx_values = dict()
        for asm in self.assemblies:
            nx_vector = asm.calculate_Nx()
            nx_values[asm.name] = nx_vector
        return nx_values

    def _calculate_auN(self) -> AUN_DICT:
        """
        Calculate auN values for each assembly

        :return: Dict of auN values per assembly
        """
        aun_values = dict()
        for asm in self.assemblies:
            aun = asm.calculate_auN()
            aun_values[asm.name] = aun
        return aun_values

    def _generate_nx_frame(self, nx_values: NX_DICT) -> pd.DataFrame:
        """
        Generate pandas frame from values of all assemblies

        :param nx_values: Dict of Nx values per assembly
        :return: Dataframe of Nx values for all assemblies
        """
        ndf = {'Nx': [], 'val': [], 'name': [], 'reference': []}
        for name, vector in nx_values.items():
            ndf['Nx'].extend(np.arange(1, 101))
            ndf['val'].extend(vector)
            ndf['name'].extend([name] * 100)
            # mark the reference assembly for downstream analyses
            if name == self.ref_name:
                refv = np.ones(100, dtype="bool")
            else:
                refv = np.zeros(100, dtype="bool")
            ndf['reference'].extend(refv)
        # transform to dataframe
        nx_df = pd.DataFrame(ndf)
        return nx_df

    def _generate_aun_frame(self, aun_values: AUN_DICT) -> pd.DataFrame:
        """
        Generate pandas frame from values of all assemblies

        :param aun_values: Dict of auN values per assembly
        :return: Dataframe of auN values for all assemblies
        """
        ndf = {'auN': [], 'name': [], 'reference': []}
        for name, value in aun_values.items():
            ndf['auN'].append(value)
            ndf['name'].append(name)
            # mark the reference assembly for downstream analyses
            refi = True if name == self.ref_name else False
            ndf['reference'].append(refi)
        # transform to dataframe
        aun_df = pd.DataFrame(ndf)
        return aun_df

    def _initialise_assemblies(self, assembly_paths: List[pathlib.Path]) -> List['Assembly']:
        """
        Load an Assembly object for each input file

        :param assembly_paths: List of input Paths
        :return: List of initialised Assembly objects
        """
        assemblies = []
        for ap in assembly_paths:
            assemblies.append(Assembly(path=ap, gsize=self.genome_size))
        return assemblies

class Assembly:

    def __init__(self, path: Union[pathlib.Path, str], gsize: int = None):
        self.path = pathlib.Path(path)
        self.name = self.path.name
        self.gsize = 0 if not gsize else gsize
        self.gzipped = is_gzipped(self.path)
        self.open_func = gzip.open if self.gzipped else open
        self.symbol = self._which_fx()
        self.lengths = self._get_sequence_lengths()

    def _which_fx(self) -> str:
        """
        read first byte to determine file type

        :return: header symbol of filetype
        """
        with self.open_func(self.path, 'rb') as f:
            symbol = str(f.read(1), 'utf-8')

        if symbol not in {'@', '>'}:
            raise FileNotFoundError("Input is neither fa nor fq")
        return symbol

    def _get_sequence_lengths(self) -> np.ndarray:
        """
        Load the sequence lengths from a file

        :return: array of sequence lengths
        """
        assert os.path.getsize(self.path) != 0
        seq_lengths = []
        with self.open_func(self.path, 'r') as fx:
            for header, seq in read_fx(fx, self.gzipped, self.symbol):
                seq_lengths.append(len(seq))
        # get lengths of all sequences
        lengths = np.array(seq_lengths)
        return lengths

    def calculate_Nx(self) -> List[int]:
        """
        Calculate Nx values for this assembly

        :return: Nx values of the assembly
        """
        if not self.gsize:
            gsize = np.sum(self.lengths)
        else:
            gsize = self.gsize
        nx = []
        # sort sequence length and calc cumulative sum
        seq_lengths_sorted = np.sort(self.lengths)[::-1]
        seq_lengths_sorted_cuml = np.cumsum(seq_lengths_sorted)
        asm_perc = np.arange(0.01, 1.01, 0.01)
        # multiply either by total contig length
        # or by reference length/genome estimate
        asm_p = asm_perc * gsize
        for i in range(len(asm_p)):
            j = 0
            try:
                while seq_lengths_sorted_cuml[j] < asm_p[i]:
                    j += 1
                nx.append(seq_lengths_sorted[j])
            except IndexError:
                nx.append(0)
        return nx

    def calculate_auN(self) -> float:
        """
        Calculate area under Nx for the assembly

        :return: auN value
        """
        if not self.gsize:
            aun = np.sum(np.power(self.lengths, 2)) / np.sum(self.lengths)
        else:
            aun = np.sum(self.lengths * (self.lengths / self.gsize))
        return aun

class Reference(Assembly):

    def genome_size(self) -> np.ndarray:
        """
        Sum of reference sequence lengths used for NGx and auNG values

        :return: Total sequence length of reference file
        """
        return np.sum(self.lengths)

def print_frame(df: pd.DataFrame, out_file: str = None) -> None:
    """
    Print a dataframe in csv format either to stdout or to file

    :param df: Pandas data frame
    :param out_file: name of an output file
    :return: None
    """
    if not out_file:
        print(df.to_csv())
    else:
        df.to_csv(out_file)

def read_fx(fh: TextIO, gz: bool, symbol: str = '>') -> Tuple[str, str]:
    """
    Yield headers and sequences of a fasta file

    :param fh: File handle of an open file connection
    :param gz: is file gzipped
    :param symbol: header symbol to determine fq or fa
    :return: Tuple of fasta header and sequence
    """
    if not gz:
        faiter = (x[1] for x in groupby(fh, lambda line: line[0] == symbol))
        for header in faiter:
            headerStr = header.__next__().strip().replace(symbol, '').split(' ')[0]
            seq = "".join(s.strip() for s in faiter.__next__())
            yield headerStr, seq
    else:
        faiter = (x[1] for x in groupby(fh, lambda line: str(line, 'utf-8')[0] == symbol))
        for header in faiter:
            headerStr = str(header.__next__(), 'utf-8').strip().replace(symbol, '').split(' ')[0]
            seq = "".join(str(s, 'utf-8').strip() for s in faiter.__next__())
            yield headerStr, seq

def convert_paths(paths: List[Union[pathlib.Path, str]]) -> List[pathlib.Path]:
    """
    Make sure that all paths in a list are pathlib.Path objects

    :param paths: List of paths
    :return: List of pathlib.Path objects
    """
    conv_paths = []
    for p in paths:
        conv_paths.append(pathlib.Path(p))
    return conv_paths

def is_gzipped(f: Union[pathlib.Path, str]) -> bool:
    """
    Check if a file is gzipped

    :param f: File path
    :return: bool
    """
    isgz = True
    with gzip.open(f, 'r') as fh:
        try:
            fh.read(1)
        except gzip.BadGzipFile:
            isgz = False
    return isgz


import csv
from collections import OrderedDict
from statistics import mean

def calculate_averages(input_file_name, output_file_name):
    f1 = open(input_file_name)
    reader = csv.reader(f1)
    for row in reader:
        name = row[0]
        grd = list()
        for grade in row[1:]:
            grd.append(float(grade))
        f2 = open(output_file_name,'a',newline='')
        writer = csv.writer(f2)
        tuple1 =  (name,mean(grd))
        writer.writerow(tuple1)
    f1.close()
    f2.close()

def calculate_sorted_averages(input_file_name, output_file_name):
    f3 = open(input_file_name)
    reader = csv.reader(f3)
    Lname = ''
    Lmean =''
    for row in reader:
        name = row[0]
        these_grade = list()
        for grade in row[1:]:
            these_grade.append(float(grade))
        Lname = Lname+name+' '
        Lmean = Lmean + str(mean(these_grade))+' '
    Lnames = Lname.split()
    Lmeans = Lmean.split()
    tup = list(zip(Lnames,Lmeans))
    od = OrderedDict(tup)
    sorted_d = sorted(od.items(),key=lambda x: x[1])
    f4 = open(output_file_name,'a',newline='')
    writer = csv.writer(f4)
    for i in sorted_d:
        writer.writerow(i)
    f3.close()
    f4.close()

def calculate_three_best(input_file_name, output_file_name):
    f5 = open (input_file_name)
    reader = csv.reader(f5)
    Lname = ''
    Lmean = ''
    for row in reader:
        name = row[0]
        these_grade= list()
        for grade in row[1:]:
            these_grade.append(float(grade))
        Lname = Lname + name+' '
        Lmean = Lmean + str(mean(these_grade))+' '
    Lnames = Lname.split()
    Lmeans = Lmean.split()
    tup = list(zip(Lnames,Lmeans))
    od1 = OrderedDict(tup)
    sorted_d1 = sorted(od1.items(),key=lambda x: x[1],reverse=True)
    ex = []
    ex.extend(sorted_d1[0:3])
    d2_ex = OrderedDict(ex)
    sorted_d2_ex = sorted(d2_ex.items(),key=lambda x: x[0])
    d3_ex = OrderedDict(sorted_d2_ex)
    sorted_d3_ex = sorted(d3_ex.items(),key=lambda x: x[1],reverse=True)
    f6 = open(output_file_name,'a',newline='')
    writer = csv.writer(f6)
    for row in sorted_d3_ex :
        writer.writerow(row)
    f5.close()
    f6.close()

def calculate_three_worst(input_file_name, output_file_name):
    f7=open(input_file_name)
    reader = csv.reader(f7)
    Lname =''
    Lmean = ''
    for row in reader:
        name = row[0]
        these_grade = list()
        for grade in row[1:]:
            these_grade.append(float(grade))
        Lname = Lname + name + ' '
        Lmean = Lmean + str(mean(these_grade)) + ' '
    Lnames = Lname.split()
    Lmeans = Lmean.split()
    tup = list(zip(Lnames,Lmeans))
    od = OrderedDict(tup)
    sorted_d = sorted(od.items(),key=lambda v: v[1])
    ex = []
    ex.extend(sorted_d[0:3])
    f8 = open(output_file_name,'a',newline='')
    writer = csv.writer(f8)
    for row in ex :
        writer.writerow(row[1:])
    f7.close()
    f8.close()

def calculate_average_of_averages(input_file_name, output_file_name):
    f9 = open(input_file_name)
    reader = csv.reader(f9)
    Lname = ''
    Lmean = ''
    for row in reader:
        name = row[0]
        these_grade = list()
        for grade in row[1:]:
            these_grade.append(float(grade))
        Lname = Lname + name +' '
        Lmean = Lmean + str(mean(these_grade))+' '
    Lnames = Lname.split()
    Lmeans = Lmean.split()
    d = dict(zip(Lnames,Lmeans))
    d_val = d.values()
    float_d_val = [float(x)for x in d_val]
    dval_mean =str(mean(float_d_val))
    f10 = open(output_file_name,'a',newline='')
    writer = csv.writer(f10)
    writer.writerow([dval_mean])
    f9.close()
    f10.close()


# Command tool to view system statisitics
from commands import Command
import psutil
import math

def convert_size(size_bytes):
   """Converts a number of bytes to its respective unit such as GB or MB"""
   if size_bytes == 0:
       return "0B"
   size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
   i = int(math.floor(math.log(size_bytes, 1024)))
   p = math.pow(1024, i)
   s = round(size_bytes / p, 2)
   return "%s %s" % (s, size_name[i])

class Stats(Command):
    command_name = "stats"

    def command(self, shell, args):
        try:
            command = args[0].lower()
        except:
            # No args were passed
            print("Current System Stats:")
            print(f"CPU Usage: {psutil.cpu_percent(interval=1)}%, Memory Usage: {psutil.virtual_memory().percent}%, Disk Usage: {psutil.disk_usage('/').percent}%")
        else:
            if command == "cpu":
                cpu_speed = psutil.cpu_freq(percpu=True)[0].current
                cpu_percents = psutil.cpu_percent(interval=1, percpu=True)
                print(f"Current CPU Speed ({cpu_speed / 1000} GHZ):")

                for i in range(psutil.cpu_count(logical=False)):
                    current_percent = cpu_percents[i]
                    print(f"CPU {i}: {current_percent}%", end=" ")
                print()
            elif command == "memory":
                memory_stats = psutil.virtual_memory()
                print("Current memory statistics:")
                print(f"Current Memory Usage: {convert_size(memory_stats.used)}, Current Memory Free: {convert_size(memory_stats.free)}, Current Memory Usage(%): {memory_stats.percent}, Total memory available: {convert_size(memory_stats.total)}")

            else:
                print("Unknown subcommand")
    def help(self):
        print("stats [SUBCOMMAND]")
        print("Allows you to view hardware stats.")
        print("\tstats cpu: Shows detailed info about CPU utilization")
        print("\tstats memory: Shows detailed information about RAM")


import pymongo
import os
from datetime import datetime
import math
import re
import pymongo
import keys  # .gitgnored file
from bson.objectid import ObjectId
from bson.son import SON
import atexit
import math

CONNECTION = None

PAGESIZE = 10

def date_now():
    """
    Needed to keep the same date in python and mongo, as mongo rounds to millisecond
    """
    d = datetime.utcnow()
    return d.replace(microsecond=math.floor(d.microsecond/1000)*1000)

CONNECTION = None

def close_connection():
    global CONNECTION
    if CONNECTION is not None:
        CONNECTION.close()

atexit.register(close_connection)

def get_connection():
    global CONNECTION
    if CONNECTION is not None:
        return CONNECTION
    else:
        mongo_db_key = os.getenv("LOCAL_DB_KEY", None)
        if mongo_db_key is None:
            exit("LOCAL_DB_KEY env variable is not set.")
        "Return mongodb connection"
        CONNECTION = pymongo.MongoClient(mongo_db_key)
        return CONNECTION

def check_run_name(name):
    connection = get_connection()
    db = connection.get_database()
    # Fastest.
    run = db.runs.find({"name": name}).limit(1).count(True)
    return run is not 0

def get_run_list():
    connection = get_connection()
    db = connection['bifrost_upgrade_test']
    # Fastest.
    runs = list(db.runs.find( {},#{"type": "routine"}, #Leave in routine
                                {"name": 1,
                                "_id": 0,
                                "samples": 1}).sort([['metadata.created_at', pymongo.DESCENDING]]))
    return runs

def get_group_list(run_name=None):
    connection = get_connection()
    db = connection.get_database()
    if run_name is not None:
        run = db.runs.find_one(
            {"name": run_name},
            {
                "_id": 0,
                "samples._id": 1
            }
        )
        if run is None:
            run_samples = []
        else:
            run_samples = run["samples"]
        sample_ids = [s["_id"] for s in run_samples]
        groups = list(db.samples.aggregate([
            {
                "$match": {
                    "_id": {"$in": sample_ids},
                }
            },
            {
                "$group": {
                    "_id": "$properties.sample_info.summary.group",
                    "count": {"$sum": 1}
                }
            }
        ]))
    else:
        groups = list(db.samples.aggregate([
            {
                "$group": {
                    "_id": "$properties.sample_info.summary.group",
                    "count": { "$sum":1 }
                }
            }
        ]))

    return groups

def get_species_list(species_source, run_name=None):
    connection = get_connection()
    db = connection.get_database()
    if species_source == "provided":
        spe_field = "properties.sample_info.summary.provided_species"
    else:
        spe_field = "properties.species_detection.summary.detected_species"
    if run_name is not None:
        run = db.runs.find_one(
            {"name": run_name},
            {
                "_id": 0,
                "samples._id": 1
            }
        )
        if run is None:
            run_samples = []
        else:
            run_samples = run["samples"]
        sample_ids = [s["_id"] for s in run_samples]
        species = list(db.samples.aggregate([
            {
                "$match": {
                    "_id": {"$in": sample_ids}
                }
            },
            {
                "$group": {
                    "_id": "$" + spe_field,
                    "count": {"$sum": 1}
                }
            },
            {
                "$sort": {"_id": 1}
            }
        ]))
    else:
        species = list(db.samples.aggregate([
            {
                "$group": {
                    "_id": "$" + spe_field,
                    "count": {"$sum": 1}
                }
            },
            {
                "$sort": {"_id": 1}
            }
        ]))
    return species

def filter_qc(qc_list):
    if qc_list is None or len(qc_list) == 0:
        return None
    qc_query = []
    for elem in qc_list:
        if elem == "Not checked":
            qc_query.append({"$and": [
                {"properties.datafiles.summary.paired_reads": {"$exists": True}},
                {"properties.stamper.summary.stamp.value": {"$exists": False}}
            ]})
        elif elem == "core facility":
            qc_query.append({"$or": [
                        {"properties.datafiles.summary.paired_reads": {"$exists": False}},
                        {"properties.stamper.summary.stamp.value": "core facility"}
                    ]
                })
        else:
            qc_query.append({"properties.stamper.summary.stamp.value": elem})
    return {"$match": {"$and": qc_query}}

def filter(run_names=None,
           species=None, species_source="species", group=None,
           qc_list=None, samples=None, pagination=None,
           sample_names=None,
           projection=None):
    if species_source == "provided":
        spe_field = "properties.provided_species"
    elif species_source == "detected":
        spe_field = "properties.detected_species"
    else:
        spe_field = "properties.species"
    connection = get_connection()
    db = connection['bifrost_upgrade_test']
    query = []
    sample_set = set()
    if sample_names is not None and len(sample_names) != 0:
        sample_names_query = []
        for s_n in sample_names:
            if s_n.startswith("/") and s_n.endswith("/"):
                sample_names_query.append(re.compile(s_n[1:-1]))
            else:
                sample_names_query.append(s_n)
        query.append({"name": {"$in": sample_names_query}})
    if samples is not None and len(samples) != 0:
        sample_set = {ObjectId(id) for id in samples}
        query.append({"_id": {"$in": list(sample_set)}})
    if run_names is not None and len(run_names) != 0:
        runs = list(db.runs.find(
            {"name": {"$in": run_names}},
            {
                "_id": 0,
                "samples._id": 1
            }
        ))
        if runs is None:
            run_sample_set = set()
        else:
            run_sample_set = {s["_id"] for run in runs for s in run['samples']}

        if len(sample_set):
            inter = run_sample_set.intersect(sample_set)
            query.append({"_id": {"$in": list(inter)}})
        else:
            query.append({"_id": {"$in": list(run_sample_set)}})
    if species is not None and len(species) != 0:

        if "Not classified" in species:
            query.append({"$or":
                [
                    {spe_field: None},
                    {spe_field: {"$in": species}},
                    {spe_field: {"$exists": False}}
                ]
            })
        else:
            query.append({spe_field: {"$in": species}})
    if group is not None and len(group) != 0:
        if "Not defined" in group:
            query.append({"$or":
                [
                    {"properties.sample_info.summary.group": None},
                    {"properties.sample_info.summary.group": {"$in": group}},
                    {"properties.sample_info.summary.group": {"$exists": False}}
                ]
            })
        else:
            query.append(
                {"properties.sample_info.summary.group": {"$in": group}})

    if pagination is not None:
        p_limit = pagination['page_size']
        p_skip = pagination['page_size'] * pagination['current_page']
    else:
        p_limit = 1000
        p_skip = 0

    skip_limit_steps = [
        {"$skip": p_skip}, {"$limit": p_limit}
    ]

    qc_query = filter_qc(qc_list)

    if len(query) == 0:
        if qc_query is None:
            match_query = {}
        else:
            match_query = qc_query["$match"]
    else:
        if qc_query is None:
            match_query = {"$and": query}
        else:
            match_query = {"$and": query + qc_query["$match"]["$and"]}
    query_result = list(db.samples.find(
        match_query, projection).sort([('name', pymongo.ASCENDING)]).skip(p_skip).limit(p_limit))

    return query_result

def get_species_QC_values(ncbi_species):
    connection = get_connection()
    db = connection.get_database('bifrost_species')
    species = db.species.find_one({"ncbi_species": ncbi_species}, {
                        "min_length": 1, "max_length": 1})
    if species is not None:
        return species
    species = db.species.find_one({"organism": ncbi_species}, {
        "min_length": 1, "max_length": 1})
    if species is not None:
        return species
    species = db.species.find_one({"group": ncbi_species}, {
        "min_length": 1, "max_length": 1})
    if species is not None:
        return species
    species = db.species.find_one({"organism": "default"}, {
        "min_length": 1, "max_length": 1})
    return species


"""
Created on Jan 4, 2017

Inception V3 2 Tower model implementation

@author: Levan Tsinadze
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

map1 = 32
map2 = 64
num_fc1 = 700  # 1028
num_fc2 = 10
reduce1x1 = 16
dropout = 0.5

STRIDES = [1, 1, 1, 1]
POOL_KERNEL = [1, 3, 3, 1]
PADDING = 'SAME'

def createWeight(shape, Name):
  """Creates weights matrix for convolution
    Args:
      shape - matrix shape
      Name - operation name
    Returns:
      weights matrix
  """
  return tf.Variable(tf.truncated_normal(shape, stddev=0.1), name=Name)

def createBias(shape, Name):
  """Creates biases matrix for convolution
    Args:
      shape - matrix shape
      Name - operation name
    Returns:
      biases matrix
  """
  return tf.Variable(tf.constant(0.1, shape=shape),
                      name=Name)

def conv2d_s1(x, W):
  """Convolution operation
    Args:
      x - layer
      W - weights
    Returns:
      convolution operation
  """
  return tf.nn.conv2d(x, W, strides=STRIDES, padding=PADDING)

def max_pool_3x3_s1(x):
  """Max-pooling operation
    Args:
      x - layer
    Returns:
      max-pooling operation
  """
  return tf.nn.max_pool(x, ksize=POOL_KERNEL, strides=STRIDES, padding=PADDING)

def inception_tower_1(x):
  """Inception Module1
    Args:
      x - input layer
    Returns:
      inception1 - concatinated Inception tower
  """
    #
    # follows input
  W_conv1_1x1_1 = createWeight([1, 1, 1, map1], 'W_conv1_1x1_1')
  b_conv1_1x1_1 = createBias([map1], 'b_conv1_1x1_1')

  # follows input
  W_conv1_1x1_2 = createWeight([1, 1, 1, reduce1x1], 'W_conv1_1x1_2')
  b_conv1_1x1_2 = createBias([reduce1x1], 'b_conv1_1x1_2')

  # follows input
  W_conv1_1x1_3 = createWeight([1, 1, 1, reduce1x1], 'W_conv1_1x1_3')
  b_conv1_1x1_3 = createBias([reduce1x1], 'b_conv1_1x1_3')

  # follows 1x1_2
  W_conv1_3x3 = createWeight([3, 3, reduce1x1, map1], 'W_conv1_3x3')
  b_conv1_3x3 = createBias([map1], 'b_conv1_3x3')

  # follows 1x1_3
  W_conv1_5x5 = createWeight([5, 5, reduce1x1, map1], 'W_conv1_5x5')
  b_conv1_5x5 = createBias([map1], 'b_conv1_5x5')

  # follows max pooling
  W_conv1_1x1_4 = createWeight([1, 1, 1, map1], 'W_conv1_1x1_4')
  b_conv1_1x1_4 = createBias([map1], 'b_conv1_1x1_4')

  conv1_1x1_1 = conv2d_s1(x, W_conv1_1x1_1) + b_conv1_1x1_1
  conv1_1x1_2 = tf.nn.relu(conv2d_s1(x, W_conv1_1x1_2) + b_conv1_1x1_2)
  conv1_1x1_3 = tf.nn.relu(conv2d_s1(x, W_conv1_1x1_3) + b_conv1_1x1_3)
  conv1_3x3 = conv2d_s1(conv1_1x1_2, W_conv1_3x3) + b_conv1_3x3
  conv1_5x5 = conv2d_s1(conv1_1x1_3, W_conv1_5x5) + b_conv1_5x5
  maxpool1 = max_pool_3x3_s1(x)
  conv1_1x1_4 = conv2d_s1(maxpool1, W_conv1_1x1_4) + b_conv1_1x1_4

  # concatenate all the feature maps and hit them with a relu
  inception1 = tf.nn.relu(tf.concat(3, [conv1_1x1_1, conv1_3x3, conv1_5x5, conv1_1x1_4]))

  return inception1

def inception_tower_2(inception_x):
  """Inception Module2
    Args:
      x - input layer
    Returns:
      inception2 - second Inception tower
  """
    #
    # follows inception1
  W_conv2_1x1_1 = createWeight([1, 1, 4 * map1, map2], 'W_conv2_1x1_1')
  b_conv2_1x1_1 = createBias([map2], 'b_conv2_1x1_1')

  # follows inception1
  W_conv2_1x1_2 = createWeight([1, 1, 4 * map1, reduce1x1], 'W_conv2_1x1_2')
  b_conv2_1x1_2 = createBias([reduce1x1], 'b_conv2_1x1_2')

  # follows inception1
  W_conv2_1x1_3 = createWeight([1, 1, 4 * map1, reduce1x1], 'W_conv2_1x1_3')
  b_conv2_1x1_3 = createBias([reduce1x1], 'b_conv2_1x1_3')

  # follows 1x1_2
  W_conv2_3x3 = createWeight([3, 3, reduce1x1, map2], 'W_conv2_3x3')
  b_conv2_3x3 = createBias([map2], 'b_conv2_3x3')

  # follows 1x1_3
  W_conv2_5x5 = createWeight([5, 5, reduce1x1, map2], 'W_conv2_5x5')
  b_conv2_5x5 = createBias([map2], 'b_conv2_5x5')

    # follows max pooling
  W_conv2_1x1_4 = createWeight([1, 1, 4 * map1, map2], 'W_conv2_1x1_4')
  b_conv2_1x1_4 = createBias([map2], 'b_conv2_1x1_4')

  conv2_1x1_1 = conv2d_s1(inception_x, W_conv2_1x1_1) + b_conv2_1x1_1
  conv2_1x1_2 = tf.nn.relu(conv2d_s1(inception_x, W_conv2_1x1_2) + b_conv2_1x1_2)
  conv2_1x1_3 = tf.nn.relu(conv2d_s1(inception_x, W_conv2_1x1_3) + b_conv2_1x1_3)
  conv2_3x3 = conv2d_s1(conv2_1x1_2, W_conv2_3x3) + b_conv2_3x3
  conv2_5x5 = conv2d_s1(conv2_1x1_3, W_conv2_5x5) + b_conv2_5x5
  maxpool2 = max_pool_3x3_s1(inception_x)
  conv2_1x1_4 = conv2d_s1(maxpool2, W_conv2_1x1_4) + b_conv2_1x1_4

  # concatenate all the feature maps and hit them with a relu
  inception2 = tf.nn.relu(tf.concat(3, [conv2_1x1_1, conv2_3x3, conv2_5x5, conv2_1x1_4]))

  return inception2

def dropout_layer(net, train=False):
  """Adds dropout layer if training
    Args:
      net - previous layer
      train - training flag
    Returns:
      out - next layer
  """

  if train:
      out = tf.nn.dropout(net, dropout)
  else:
      out = net

  return out

def model(x, size, train=False):
  """Inception model
    Args:
      x - model input
      size - image size
      train - training flag
    Returns:
      out - model output
  """

  # Fully connected layers
    # since padding is same, the feature map with there will be 4 28*28*map2
  W_fc1 = createWeight([size * size * (4 * map2), num_fc1], 'W_fc1')
  b_fc1 = createBias([num_fc1], 'b_fc1')

  W_fc2 = createWeight([num_fc1, num_fc2], 'W_fc2')
  b_fc2 = createBias([num_fc2], 'b_fc2')

  inception1 = inception_tower_1(x)
  inception2 = inception_tower_2(inception1)

# flatten features for fully connected layer
  inception2_flat = tf.reshape(inception2, [-1, size * size * 4 * map2])

  # Fully connected layers
  h_fc0 = tf.nn.relu(tf.matmul(inception2_flat, W_fc1) + b_fc1)
  h_fc1 = dropout_layer(h_fc0, train=train)
  out = tf.matmul(h_fc1, W_fc2) + b_fc2

  return out

def interface(x, size):
  """Network interface
    Args:
      x - model input
      size - image size
    Returns:
      prediction - predicted value
  """

  linear_pred = model(x, size)
  prediction = tf.nn.softmax(linear_pred)

  return prediction


"""Flame graph module."""
import inspect
import runpy
import signal
import time

from collections import defaultdict
from vprof import base_profiler

_SAMPLE_INTERVAL = 0.001

class _StatProfiler:
    """Statistical profiler.

    Samples call stack at regular intervals specified by _SAMPLE_INTERVAL.
    """

    def __init__(self):
        self._stats = defaultdict(int)
        self._start_time = None
        self.base_frame = None
        self.run_time = None

    def __enter__(self):
        """Enables statistical profiler."""
        signal.signal(signal.SIGPROF, self.sample)
        signal.setitimer(signal.ITIMER_PROF, _SAMPLE_INTERVAL)
        self._start_time = time.time()
        return self

    def __exit__(self, exc_type, exc_val, exc_tbf):
        """Disables statistical profiler."""
        self.run_time = time.time() - self._start_time
        signal.setitimer(signal.ITIMER_PROF, 0)

    def sample(self, signum, frame):  #pylint: disable=unused-argument
        """Samples current stack and writes result in self._stats.

        Args:
            signum: Signal that activates handler.
            frame: Frame on top of the stack when signal is handled.
        """
        stack = []
        while frame and frame != self.base_frame:
            stack.append((
                frame.f_code.co_name,
                frame.f_code.co_filename,
                frame.f_code.co_firstlineno))
            frame = frame.f_back
        self._stats[tuple(stack)] += 1
        signal.setitimer(signal.ITIMER_PROF, _SAMPLE_INTERVAL)

    @staticmethod
    def _insert_stack(stack, sample_count, call_tree):
        """Inserts a stack into the call tree.

        Args:
            stack: Call stack.
            sample_count: Sample count of call stack.
            call_tree: Call tree.
        """
        curr_level = call_tree
        for func in stack:
            next_level_index = {
                node['stack']: node for node in curr_level['children']}
            if func not in next_level_index:
                new_node = {'stack': func, 'children': [], 'sampleCount': 0}
                curr_level['children'].append(new_node)
                curr_level = new_node
            else:
                curr_level = next_level_index[func]
        curr_level['sampleCount'] = sample_count

    def _fill_sample_count(self, node):
        """Counts and fills sample counts inside call tree."""
        node['sampleCount'] += sum(
            self._fill_sample_count(child) for child in node['children'])
        return node['sampleCount']

    @staticmethod
    def _get_percentage(sample_count, total_samples):
        """Return percentage of sample_count in total_samples."""
        if total_samples != 0:
            return 100 * round(float(sample_count) / total_samples, 3)
        return 0.0

    def _format_tree(self, node, total_samples):
        """Reformats call tree for the UI."""
        funcname, filename, _ = node['stack']
        sample_percent = self._get_percentage(
            node['sampleCount'], total_samples)
        color_hash = base_profiler.hash_name('%s @ %s' % (funcname, filename))
        return {
            'stack': node['stack'],
            'children': [self._format_tree(child, total_samples)
                         for child in node['children']],
            'sampleCount': node['sampleCount'],
            'samplePercentage': sample_percent,
            'colorHash': color_hash
        }

    @property
    def call_tree(self):
        """Returns call tree."""
        call_tree = {'stack': 'base', 'sampleCount': 0, 'children': []}
        for stack, sample_count in self._stats.items():
            self._insert_stack(reversed(stack), sample_count, call_tree)
        self._fill_sample_count(call_tree)
        if not call_tree['children']:
            return {}
        return self._format_tree(
            call_tree['children'][0], call_tree['sampleCount'])

class FlameGraphProfiler(base_profiler.BaseProfiler):
    """Statistical profiler wrapper.

    Runs statistical profiler and returns collected stats.
    """

    def _profile_package(self):
        """Runs statistical profiler on a package."""
        with _StatProfiler() as prof:
            prof.base_frame = inspect.currentframe()
            try:
                runpy.run_path(self._run_object, run_name='__main__')
            except SystemExit:
                pass

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'timestamp': int(time.time())
        }

    def profile_package(self):
        """Runs package profiler in a separate process."""
        return base_profiler.run_in_separate_process(self._profile_package)

    def _profile_module(self):
        """Runs statistical profiler on a module."""
        with open(self._run_object, 'rb') as srcfile, _StatProfiler() as prof:
            code = compile(srcfile.read(), self._run_object, 'exec')
            prof.base_frame = inspect.currentframe()
            try:
                exec(code, self._globs, None)
            except SystemExit:
                pass

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'timestamp': int(time.time())
        }

    def profile_module(self):
        """Runs module profiler in a separate process."""
        return base_profiler.run_in_separate_process(self._profile_module)

    def profile_function(self):
        """Runs statistical profiler on a function."""
        with _StatProfiler() as prof:
            result = self._run_object(*self._run_args, **self._run_kwargs)

        call_tree = prof.call_tree
        return {
            'objectName': self._object_name,
            'sampleInterval': _SAMPLE_INTERVAL,
            'runTime': prof.run_time,
            'callStats': call_tree,
            'totalSamples': call_tree.get('sampleCount', 0),
            'result': result,
            'timestamp': int(time.time())
        }


import json
import os

from datetime import timedelta, date, datetime
from flask import render_template, request, redirect, Response, send_from_directory, session, url_for, abort
from flask.views import MethodView
from flask_admin.contrib import sqla
from flask_security import current_user

from clocking.models import db, Person, Entry, Address
from clocking.api.forms import PersonForm, MacForm, SelectForm, LoginForm
from clocking.api.generate_report import generate_report

from instance.settings import REPORT_DIR, REPORT_FILE

class ProtectedModelView(sqla.ModelView):

    def is_accessible(self):
        if not current_user.is_active or not current_user.is_authenticated:
            return False

        if current_user.has_role('superuser'):
            return True

        return False

    def _handle_view(self, name, **kwargs):
        """
        Override builtin _handle_view in order to redirect users when a view is not accessible.
        """
        if not self.is_accessible():
            if current_user.is_authenticated:
                abort(403)
            else:
                return redirect(url_for('security.login', next=request.url))

class PersonAddView(MethodView):

    def get(self):
        form = PersonForm()
        return render_template('add.html', form=form)

    def post(self):
        data = {}
        form = PersonForm(request.form)
        if form.validate():
            person = form.save()
            data['html'] = render_template('bits/mac_form.html', person=person)
            data['status'] = 'success'
        else:
            data['html'] = render_template('bits/form_errors.html', form=form)
            data['status'] = 'error'

        return Response(json.dumps(data), content_type='application/json')

class MacAddView(MethodView):

    def post(self):
        data = {}
        form = MacForm(request.form)
        if form.validate():
            address = form.save()
            data['html'] = render_template('bits/mac_listing.html',
                                           address=address)
            data['status'] = 'success'
        else:
            data['html'] = render_template('bits/form_errors.html',
                                           form=form)
            data['status'] = 'error'

        return Response(json.dumps(data), content_type='application/json')

class MacDeleteView(MethodView):

    def get(self, mac_address):
        address = db.session.query(Address).get(mac_address)
        return render_template('delete.html', address=address)

    def post(self, mac_address):
        address = db.session.query(Address).get(mac_address)
        address.deleted = True
        try:
            db.session.commit()
        except:
            db.session.rollback()

        persons = filter_persons_addresses()
        return render_template('people.html', persons=persons)

class PersonEditView(MethodView):

    def get(self, person_id):
        person = db.session.query(Person).get(person_id)
        person_form = PersonForm()
        mac_form = MacForm()
        return render_template('edit.html', person_form=person_form,
                               mac_form=mac_form, person=person)

    def post(self, person_id):
        data = {}
        form = PersonForm(request.form)
        if form.validate():
            person = form.save(person_id)
            data['html'] = render_template('bits/person_edit.html',
                                           person=person)
            data['status'] = 'success'
        else:
            data['html'] = render_template('bits/person_edit.html',
                                           form=form)
            data['status'] = 'error'

        return Response(json.dumps(data), content_type='application/json')

class PersonListView(MethodView):

    def get(self):
        persons = filter_persons_addresses()
        return render_template('people.html', persons=persons)

class PersonClockingView(MethodView):

    def get(self):
        form = SelectForm()
        start_date, end_date = date.today(), date.today()
        if request.args:
            form = SelectForm(request.args)
            if form.validate():
                start_date = form.data.get('start_date')
                end_date = form.data.get('end_date')
        days = get_days(start_date, end_date)
        entries = get_all_entries(days)
        return render_template('clocking.html', form=form, entries=entries,
                               start_date=start_date, end_date=end_date)

class DownloadView(MethodView):

    def get(self, start_date, end_date):
        start_date = datetime.strptime(start_date, '%Y-%m-%d')
        end_date = datetime.strptime(end_date, '%Y-%m-%d')
        days = get_days(start_date, end_date)
        entries = get_all_entries(days)
        generate_report(entries)
        file = send_from_directory(directory=REPORT_DIR,
                                   filename=REPORT_FILE,
                                   as_attachment=True)
        os.remove(REPORT_DIR + REPORT_FILE)
        return file

class AboutView(MethodView):

    def get(self):
        return render_template('about.html')

def filter_persons_addresses():
    persons = Person.query.join(Address).filter(
        Address.deleted==False).order_by(Person.first_name)
    for person in persons:
        person.addresses = [address for address in person.addresses if
                            address.deleted is False]
    return persons

def get_days(start_date, end_date):
    interval = end_date - start_date
    days = []
    for i in range(interval.days + 1):
        days.append(start_date + timedelta(days=i))
    return days

def get_all_entries(days):
    entries = []
    for day in days:
        entry = {}
        entry['day'] = day
        entry['entries_by_day'] = get_entries_by_day(day).all()
        if entry['entries_by_day']:
            entries.append(entry)
    return entries

def get_entries_by_day(day):
    entries = Entry.query
    return entries.filter(Entry.startdate >= day).filter(
        Entry.startdate <= day + timedelta(hours=24)).order_by(
        Entry.startdate)


import json, os, platform, subprocess, sys, time
from datetime import datetime, timedelta
from operator import itemgetter

isPy3 = True

try:
    from colorama import init

    init()
except ImportError as e:
    pass

scripts_lib_path = os.path.join(os.path.dirname(__file__), "lib")
this_platform_system = str(platform.system())
platform_lib_path = os.path.join(scripts_lib_path, this_platform_system)
if os.path.exists(platform_lib_path):
    if platform_lib_path not in sys.path:
        sys.path.append(platform_lib_path)

import util

python_exe = sys.executable
python_ver = platform.python_version()

class bcolors:
    HEADER = "\033[95m"
    OKBLUE = "\033[94m"
    OKGREEN = "\033[92m"
    WARNING = "\033[93m"
    FAIL = "\033[91m"
    ENDC = "\033[0m"
    BOLD = "\033[1m"
    UNDERLINE = "\033[4m"
    BACKGROUND = "\033[42m"
    ITALIC = "\033[3m"

bold_start = bcolors.BOLD
bold_end = bcolors.ENDC
italic_start = bcolors.ITALIC
italic_end = bcolors.ENDC
table_header_style = bcolors.BOLD + bcolors.BACKGROUND
error_start = bcolors.FAIL

def format_help(p_input):
    inp = str(p_input)
    inp_lst = inp.split()

    p_1st = None
    if inp_lst:
        p_1st = str(inp_lst[0])

    if p_1st in ("#", "##", "###"):
        skip_len = len(p_1st) + 1
        inp = inp[skip_len:]
        inp = inp.replace("`", "")
        inp = bold_start + str(inp.upper()) + bold_end

    elif inp == "```":
        return None

    else:
        inp = inp.replace(" # ", italic_start + " # ")
        inp = "  " + inp + " " + italic_end

    return inp

def get_pip_ver():
    try:
        import pip

        return pip.__version__
    except ImportError as e:
        pass
    return "None"

def cli_unicode(p_str, p_encoding, errors="ignore"):
    return str(p_str)

try:
    test_unicode = unicode("test")
except NameError as e:
    unicode = cli_unicode

def check_output_wmic(p_cmds):
    out1 = subprocess.check_output(p_cmds)
    try:
        out2 = str(out1, "utf-8")
    except:
        out2 = str(out1)
    out3 = out2.strip().split("\n")[1]
    return out3

def top(display=True, isJson=False):
    try:
        import psutil
    except ImportError as e:
        util.exit_message("Missing psutil module", 1)

    current_timestamp = int(time.mktime(datetime.utcnow().timetuple()))
    jsonDict = {}
    procs = []
    for p in psutil.process_iter():
        try:
            p = p.as_dict(
                attrs=[
                    "pid",
                    "username",
                    "cpu_percent",
                    "memory_percent",
                    "cpu_times",
                    "name",
                ]
            )
        except (psutil.NoSuchProcess, IOError, OSError) as e:
            pass
        else:
            procs.append(p)

    if not display:
        return

    processes = sorted(procs, key=lambda p: p["cpu_percent"], reverse=True)

    network_usage = psutil.net_io_counters()
    jsonDict["kb_sent"] = network_usage.bytes_sent / 1024
    jsonDict["kb_recv"] = network_usage.bytes_recv / 1024

    cpu = psutil.cpu_times_percent(percpu=False)
    iowait = ""
    if util.get_platform() == "Linux":
        jsonDict["iowait"] = str(cpu.iowait)
        iowait = "," + str(cpu.iowait).rjust(5) + "%wa"

    jsonDict["current_timestamp"] = current_timestamp
    jsonDict["cpu_user"] = str(cpu.user)
    jsonDict["cpu_system"] = str(cpu.system)
    jsonDict["cpu_idle"] = str(cpu.idle)
    if not isJson:
        print(
            "CPU(s):"
            + str(cpu.user).rjust(5)
            + "%us,"
            + str(cpu.system).rjust(5)
            + "%sy,"
            + str(cpu.idle).rjust(5)
            + "%id"
            + iowait
        )

    disk = psutil.disk_io_counters(perdisk=False)
    read_kb = disk.read_bytes / 1024
    write_kb = disk.write_bytes / 1024
    jsonDict["kb_read"] = str(read_kb)
    jsonDict["kb_write"] = str(write_kb)
    if not isJson:
        print("DISK: kB_read " + str(read_kb) + ", kB_written " + str(write_kb))

    uptime = datetime.now() - datetime.fromtimestamp(psutil.boot_time())
    str_uptime = str(uptime).split(".")[0]
    line = ""
    uname_len = 8
    av1, av2, av3 = os.getloadavg()
    str_loadavg = "%.2f %.2f %.2f  " % (av1, av2, av3)
    line = bold_start + "Load average: " + bold_end + str_loadavg
    jsonDict["load_avg"] = str(str_loadavg)
    line = line + bold_start + "Uptime:" + bold_end + " " + str_uptime
    jsonDict["uptime"] = str(str_uptime)
    if not isJson:
        print(line)

    i = 0
    my_pid = os.getpid()
    if not isJson:
        print("")
        print(
            bold_start
            + "    PID "
            + "USER".ljust(uname_len)
            + "   %CPU %MEM      TIME+ COMMAND"
            + bold_end
        )

    jsonList = []
    for pp in processes:
        if pp["pid"] == my_pid:
            continue
        i += 1
        if i > 10:
            break

        # TIME+ column shows process CPU cumulative time and it
        # is expressed as: "mm:ss.ms"

        ctime = timedelta(seconds=sum(pp["cpu_times"]))
        ctime_mm = str(ctime.seconds // 60 % 60)
        ctime_ss = str(int(ctime.seconds % 60)).zfill(2)
        ctime_ms = str(ctime.microseconds)[:2].ljust(2, str(0))
        ctime = "{0}:{1}.{2}".format(ctime_mm, ctime_ss, ctime_ms)

        username = pp["username"][:uname_len]
        if isJson:
            pp["username"] = username
            pp["ctime"] = ctime
            pp["cpu_percent"] = float(pp["cpu_percent"])
            pp["memory_percent"] = float(round(pp["memory_percent"], 1))
            jsonList.append(pp)
        else:
            print(
                str(pp["pid"]).rjust(7)
                + " "
                + username.ljust(uname_len)
                + " "
                + str(pp["cpu_percent"]).rjust(6)
                + " "
                + str(round(pp["memory_percent"], 1)).rjust(4)
                + " "
                + str(ctime).rjust(10)
                + " "
                + pp["name"]
            )
    if isJson:
        jsonDict["top"] = jsonList
        print(json.dumps([jsonDict]))
    else:
        print("")

def list(p_json, p_cat, p_comp, p_ver, p_port, p_status, p_kount):
    lst = " "
    if p_kount > 1:
        lst = ","
    if p_json:
        lst = (
            lst
            + '{"category": "'
            + p_cat.rstrip()
            + '",'
            + ' "component": "'
            + p_comp.rstrip()
            + '",'
            + ' "version": "'
            + p_ver.rstrip()
            + '",'
            + ' "port": "'
            + p_port.rstrip()
            + '",'
            + ' "status": "'
            + p_status.rstrip()
            + '"}'
        )
        print(lst)
        return

    print(p_comp + "  " + p_ver + "  " + p_port + "  " + p_status)

def status(p_json, p_comp, p_ver, p_state, p_port, p_kount):
    status = " "
    if p_kount > 1:
        status = ","
    if p_json:
        jsonStatus = {}
        jsonStatus["component"] = p_comp
        jsonStatus["version"] = p_ver
        jsonStatus["state"] = p_state
        if p_port != "" and int(p_port) > 1:
            jsonStatus["port"] = p_port
        category = util.get_comp_category(p_comp)
        if category:
            jsonStatus["category"] = category
        elif p_comp.startswith == "pgdg":
            jsonStatus["category"] = 1
        print(status + json.dumps(jsonStatus))
        return

    app_ver = p_comp + "-" + p_ver
    app_ver = app_ver + (" " * (35 - len(app_ver)))

    if p_state in ("Running", "Stopped") and int(p_port) > 1:
        on_port = " on port " + p_port
    else:
        on_port = ""

    # print(app_ver + "(" + p_state + on_port + ")")
    print(p_comp + " " + p_state.lower() + on_port)

def info(p_json, p_home, p_repo, print_flag=True):
    (
        cloud_name,
        cloud_platform,
        instance_id,
        flavor,
        region,
        az,
        private_ip,
    ) = util.get_cloud_info()

    p_user = util.get_user()
    p_is_admin = util.is_admin()
    pip_ver = get_pip_ver()
    os_arch = util.get_arch()

    this_os = ""
    this_uname = str(platform.system())[0:7]
    if private_ip > "":
        host_ip = private_ip
    else:
        host_ip = util.get_host_ip()
    wmic_path = (
        os.getenv("SYSTEMROOT", "")
        + os.sep
        + "System32"
        + os.sep
        + "wbem"
        + os.sep
        + "wmic"
    )
    host_display = util.get_host_short()

    ## Check the OS & Resources ########################################
    plat = util.get_os()
    glibcV = util.get_glibc_version()

    os_major_ver = ""
    java_major_ver = ""
    java_ver = ""

    if this_uname == "Darwin":
        mem_mb = util.get_mem_mb()
        system_memory_in_kbytes = mem_mb * 1024
        system_memory_in_gb = mem_mb / 1024.0
        system_cpu_cores = util.get_cpu_cores()
        cpu_model = util.getoutput("/usr/sbin/sysctl -n machdep.cpu.brand_string")
        prod_name = util.getoutput("sw_vers -productName")
        prod_version = util.getoutput("sw_vers -productVersion")
        this_os = prod_name + " " + prod_version
    elif this_uname == "Linux":
        mem_mb = util.get_mem_mb()
        system_memory_in_kbytes = mem_mb * 1024
        system_memory_in_gb = mem_mb / 1024.0
        system_cpu_cores = util.get_cpu_cores()
        cpu_model = util.getoutput(
            "grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2"
        )
        os_major_ver = util.getoutput(
            "cat /etc/os-release | grep VERSION_ID | cut -d= -f2 | tr -d '\"'"
        )
        if cpu_model == "":
            cpu_model = "ARM"
        if os.path.exists("/etc/redhat-release"):
            this_os = util.getoutput("cat /etc/redhat-release")
        elif os.path.exists("/etc/system-release"):
            this_os = util.getoutput("cat /etc/system-release")
        elif os.path.exists("/etc/lsb-release"):
            this_os = util.getoutput(
                "cat /etc/lsb-release | grep DISTRIB_DESCRIPTION | cut -d= -f2 | tr -d '\"'"
            )
        elif os.path.exists("/etc/os-release"):
            this_os = util.getoutput(
                "cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '\"'"
            )
        [java_major_ver, java_ver] = util.get_java_ver()

    if system_memory_in_gb > 0.6:
        round_mem = round(system_memory_in_gb)
    else:
        round_mem = round(system_memory_in_gb, 1)

    mem = str(round_mem) + " GB"

    cores = str(system_cpu_cores)

    cpu = cpu_model.strip()
    cpu = cpu.replace("(R)", "")
    cpu = cpu.replace("(TM)", "")
    cpu = cpu.replace(" CPU ", " ")

    os2 = this_os.replace(" release ", " ")
    os2 = os2.replace(" (Final)", "")
    os2 = os2.replace(" (Core)", "")

    ver = util.get_version()
    [last_update_utc, last_update_local, unique_id] = util.read_hosts("localhost")
    if last_update_local:
        last_upd_dt = datetime.strptime(last_update_local, "%Y-%m-%d %H:%M:%S")
        time_diff = int(util.timedelta_total_seconds(datetime.now() - last_upd_dt))
        last_update_readable = util.get_readable_time_diff(str(time_diff), precision=2)

    versions_sql = util.get_versions_sql()
    perl_ver = util.get_perl_ver()

    os_pkg_mgr = util.get_pkg_mgr()

    if p_json:
        infoJsonArray = []
        infoJson = {}
        infoJson["version"] = ver
        infoJson["home"] = p_home
        infoJson["user"] = p_user
        infoJson["host"] = host_display
        infoJson["host_short"] = util.get_host_short()
        infoJson["host_long"] = util.get_host()
        infoJson["host_ip"] = host_ip
        infoJson["os"] = unicode(
            str(os2), sys.getdefaultencoding(), errors="ignore"
        ).strip()
        infoJson["os_pkg_mgr"] = os_pkg_mgr
        infoJson["os_major_ver"] = os_major_ver
        infoJson["platform"] = unicode(
            str(plat), sys.getdefaultencoding(), errors="ignore"
        ).strip()
        infoJson["arch"] = os_arch
        infoJson["mem"] = round_mem
        infoJson["cores"] = system_cpu_cores
        infoJson["cpu"] = cpu
        infoJson["last_update_utc"] = last_update_utc
        if last_update_local:
            infoJson["last_update_readable"] = last_update_readable
        infoJson["unique_id"] = unique_id
        infoJson["repo"] = p_repo
        infoJson["versions_sql"] = versions_sql
        infoJson["system_memory_in_kb"] = system_memory_in_kbytes
        infoJson["python_ver"] = python_ver
        infoJson["python_exe"] = python_exe
        if pip_ver != "None":
            infoJson["pip_ver"] = pip_ver
        infoJson["perl_ver"] = perl_ver
        infoJson["java_ver"] = java_ver
        infoJson["java_major_ver"] = java_major_ver
        infoJson["glibc_ver"] = glibcV
        infoJson["region"] = region
        infoJson["az"] = az
        infoJson["instance_id"] = instance_id
        infoJson["flavor"] = flavor
        infoJson["private_ip"] = private_ip
        infoJsonArray.append(infoJson)
        if print_flag:
            print(json.dumps(infoJsonArray, sort_keys=True, indent=2))
            return
        else:
            return infoJson

    if p_is_admin:
        admin_display = " (Admin)"
    else:
        admin_display = ""

    langs = "Python v" + python_ver
    if perl_ver > "":
        langs = langs + " | Perl v" + perl_ver
    if java_ver > "":
        langs = langs + " | Java v" + java_ver

    ## util.validate_distutils_click(False)

    if glibcV <= " ":
        glibc_v_display = ""
    else:
        glibc_v_display = " glibc-" + glibcV + "-"

    print(bold_start + ("#" * 70) + bold_end)
    print(bold_start + "#  pgEdge CTL: " + bold_end + "v" + ver + "  " + p_home)
    print(
        bold_start
        + "# User & Host: "
        + bold_end
        + p_user
        + admin_display
        + "  "
        + host_display
        + "  "
        + host_ip
    )
    print(
        bold_start
        + "#          OS: "
        + bold_end
        + os2.rstrip()
        + " "
        + glibc_v_display
        + os_arch
    )
    print(
        bold_start + "#     Machine: " + bold_end + mem + ", " + cores + " vCPU, " + cpu
    )
    if instance_id > "" and not cloud_name == "unknown":
        print(
            bold_start
            + "#  Cloud Info: "
            + bold_end
            + f"{cloud_name}  {cloud_platform}  {instance_id}  {flavor}  {az}"
        )

    print(bold_start + "#       Langs: " + bold_end + langs)

    print(bold_start + "#    Repo URL: " + bold_end + p_repo)

    if not last_update_local:
        last_update_local = "None"

    print(bold_start + "# Last Update: " + bold_end + str(last_update_local))
    print(bold_start + ("#" * 70) + bold_end)

def info_component(p_comp_dict, p_kount):
    if p_kount > 1:
        print(bold_start + ("-" * 90) + bold_end)

    print(
        bold_start
        + "     Project: "
        + bold_end
        + p_comp_dict["project"]
        + " ("
        + p_comp_dict["project_url"]
        + ")"
    )

    print(
        bold_start
        + "   Component: "
        + bold_end
        + p_comp_dict["component"]
        + " "
        + p_comp_dict["version"]
        + " ("
        + p_comp_dict["proj_description"]
        + ")"
    )

    if p_comp_dict["port"] > 1:
        print(bold_start + "        port: " + bold_end + str(p_comp_dict["port"]))

    if p_comp_dict["datadir"] > "":
        print(bold_start + "     datadir: " + bold_end + p_comp_dict["datadir"])

    if p_comp_dict["logdir"] > "":
        print(bold_start + "      logdir: " + bold_end + p_comp_dict["logdir"])

    if p_comp_dict["autostart"] == "on":
        print(bold_start + "   autostart: " + bold_end + p_comp_dict["autostart"])

    if p_comp_dict["svcuser"] > "" and util.get_platform() == "Linux":
        print(bold_start + "     svcuser: " + bold_end + p_comp_dict["svcuser"])

    if ("status" in p_comp_dict) and ("up_time" in p_comp_dict):
        print(
            bold_start
            + "      status: "
            + bold_end
            + p_comp_dict["status"]
            + bold_start
            + " for "
            + bold_end
            + p_comp_dict["up_time"]
        )
    else:
        if "status" in p_comp_dict:
            print(bold_start + "      status: " + bold_end + p_comp_dict["status"])
        if "up_time" in p_comp_dict:
            print(bold_start + "    up since: " + bold_end + p_comp_dict["up_time"])

    if "data_size" in p_comp_dict:
        print(bold_start + "   data size: " + bold_end + p_comp_dict["data_size"])

    if "connections" in p_comp_dict:
        print(bold_start + " connections: " + bold_end + p_comp_dict["connections"])

    print(
        bold_start
        + "Release Date: "
        + bold_end
        + p_comp_dict["release_date"]
        + bold_start
        + "  Stage: "
        + bold_end
        + p_comp_dict["stage"]
    )

    if p_comp_dict["platform"] > "":
        print(
            bold_start
            + "Supported On: "
            + bold_end
            + "["
            + p_comp_dict["platform"]
            + "]"
        )

    if p_comp_dict["pre_reqs"] > "":
        print(bold_start + "   Pre Req's: " + bold_end + p_comp_dict["pre_reqs"])

    print(bold_start + "     License: " + bold_end + p_comp_dict["license"])

    is_installed = str(p_comp_dict["is_installed"])
    if str(is_installed) == "0":
        is_installed = "NO"

    print(
        bold_start
        + "   IsCurrent: "
        + bold_end
        + str(p_comp_dict["is_current"])
        + bold_start
        + "  IsInstalled: "
        + bold_end
        + is_installed
    )

def format_data_to_table(
    data,
    keys,
    header=None,
    error_key=None,
    error_msg_column=None,
    sort_by_key=None,
    sort_order_reverse=False,
):
    """Takes a list of dictionaries, formats the data, and returns
    the formatted data as a text table.

    Required Parameters:
        data - Data to process (list of dictionaries). (Type: List)
        keys - List of keys in the dictionary. (Type: List)

    Optional Parameters:
        header - The table header. (Type: List)
        sort_by_key - The key to sort by. (Type: String)
        sort_order_reverse - Default sort order is ascending, if
            True sort order will change to descending. (Type: Boolean)
    """
    # Sort the data if a sort key is specified (default sort order
    # is ascending)
    if sort_by_key:
        data = sorted(data, key=itemgetter(sort_by_key), reverse=sort_order_reverse)

    # If header is not empty, add header to data
    if header:
        # Get the length of each header and create a divider based
        # on that length
        header_divider = []
        for name in header:
            header_divider.append("-" * len(name))

        # Create a list of dictionary from the keys and the header and
        # insert it at the beginning of the list. Do the same for the
        # divider and insert below the header.
        # header_divider = dict(zip(keys, header_divider))
        # data.insert(0, header_divider)
        header = dict(zip(keys, header))
        data.insert(0, header)

    column_widths = []
    for key in keys:
        column_widths.append(max(len(str(column[key])) for column in data) + 2)

    # Create a tuple pair of key and the associated column width for it
    key_width_pair = zip(keys, column_widths)
    key_length = len(keys)

    str_format = ("%-*s " * len(keys)).strip() + "\n"
    formatted_data = ""

    for element in data:
        data_to_format = []
        s = 0
        key_width_pair = zip(keys, column_widths)
        # Create a tuple that will be used for the formatting in
        # width, value format
        for pair in key_width_pair:
            dataStr = str(element[pair[0]])
            spaces = " " * ((int(float(pair[1])) - len(dataStr)) - 2)
            if s < key_length - 1:
                spaces = spaces + " |"

            if dataStr in header.values():
                if s == 0:
                    dataStr = table_header_style + dataStr
                dataStr = dataStr + spaces
                if s == key_length - 1:
                    dataStr = dataStr + bold_end
                s = s + 1
            elif error_key and error_msg_column:
                if (
                    pair[0] in error_msg_column
                    and element.get(error_key[0]) == error_key[1]
                ):
                    dataStr = error_start + dataStr + bold_end

            data_to_format.append(pair[1])
            data_to_format.append(dataStr)
        formatted_data += str_format % tuple(data_to_format)
    return formatted_data


#!/usr/bin/python3
"""
 江西水库信息
 Author: cg
 Date: 2020/7/11 15:58
"""

import json
import urllib.request
from crawl.spiders.base_spider import BaseSpider
from operate.schedulermgr import SchedulerMgr
from db.mapping.jiangxi_reservoir.jiangxi_reservoir import JiangxiReservoir
from db.mapping.jiangxi_reservoir.jiangxi_reservoir_item import JiangxiReservoirItem
from db.mapping.jiangxi_reservoir.jiangxi_reservoir_item_place import JiangxiReservoirItemPlace
from util import time_util
from sanic.log import logger
from sanic.log import error_logger

class JiangxiReservoirSpider(BaseSpider):

    # 接口地址
    _base_url = "http://111.75.205.67:7080/syq/reservoirmap/reservoirMapHandler"

    # key: spider ts, value: 每次爬取到的数据, 封装在JiangxiRiverItem中
    _data_ = dict()

    def __init__(self):
        super().__init__(self._base_url, True)

    def run(self):
        # 每小时的10, 40分钟30秒执行一次
        SchedulerMgr.add_job_cron(self.request_begin, day_of_week='*', hour='*', minute='15,45', second='30')

    def request(self):
        res = urllib.request.urlopen(self.client)
        res_str = res.read().decode("utf-8")
        res.close()
        res_dict = json.loads(res_str)
        logger.info("请求得到数据: " + str(res_dict))
        suc = self.operate_data(res_dict)
        if not suc:
            error_logger.error("请求处理出错 " + self._base_url)

    def operate_data(self, res_dict):
        if 'data' not in res_dict:
            logger.error("返回结果中不包含'data'")
            return False
        dict_data = res_dict['data']
        cur_ts = time_util.getcurrent_ts_millis()
        data_item = self.operate_item_data(cur_ts, dict_data)
        self._data_[cur_ts] = data_item
        return True

    def operate_item_data(self, cur_ts, dict_data):
        item = JiangxiReservoirItem()
        item.spiderTs = cur_ts
        item.upCodes = dict_data['upCodes']
        item.warnStatNum = dict_data['overTopFLZSize']
        item.warnStatInfo = self.operate_item_space(dict_data['overTopFLZ'])
        item.desc = dict_data['summarize']
        item.totalStatInfo = self.operate_item_space(dict_data['rows'])
        return item

    def operate_item_space(self, warn_list):
        res = list()
        for item in warn_list:
            data = self.build_space_info(item)
            res.append(data.__dict__)
        return res

    @staticmethod
    def build_space_info(dict_info):
        item_data = JiangxiReservoirItemPlace()
        item_data.hTM = dict_info['hTM']
        item_data.county = dict_info['county']
        item_data.level = dict_info['rz']
        item_data.enterCapacity = dict_info['inq']
        item_data.outCapacity = dict_info['otq']
        item_data.beginLevel = dict_info['ffsltdz']
        item_data.afterLevel = dict_info['fsltdz']
        item_data.exceedLevel = dict_info['cfsltdz']
        item_data.stcd = dict_info['stcd']
        item_data.stnm = dict_info['stnm']
        item_data.tm = dict_info['tm']
        item_data.style = dict_info['style']
        item_data.w = dict_info['w']
        item_data.blrz = dict_info['blrz']
        item_data.fsltdz = dict_info['fsltdz']
        return item_data

    def get_collection(self):
        return JiangxiReservoir


from nltk.compat import gutenberg

gutenberg.fileids()

dir(gutenberg)

JM_book = gutenberg.words('milton-paradise.txt')
len(JM_book)

sample_JM = JM_book[100:150]
sample_JM
len(sample_JM)

set_JM = set(sample_JM)
set_JM
len(set(set_JM))
type(set_JM)
sorteSet_JM = sorted(set_JM)
sorteSet_JM
len(sorteSet_JM)

print("milton-paradise.txt has: ", len(JM_book))
len(set(JM_book))

curso2/2 Organizing Words by Set and Sorted.py


import boto3
from colorama import Fore, Back, Style
from functions import create_client

def help():
    print(Fore.YELLOW + "\n================================================================================================" + Style.RESET_ALL)
    print("[+] Module Description:\n")
    print("\tThis module will enumerate the S3 buckets available in the account.")
    print("\tIt will print the found buckets and give you the option to enumerate.")
    print("\tthe objects on those buckets.\n")

    print("\tIf you choose to enumerate objects on the found buckets, it will")
    print("\tprint the list of objects from each bucket.")
    print(Fore.YELLOW + "================================================================================================" + Style.RESET_ALL)

def create_s3_client(botoconfig, session):
    client = create_client.Client(botoconfig, session, 's3')

    return client.create_aws_client()

def list_buckets(client):
    response = client.list_buckets()

    return response

def parse_bucket_data(bucket_data):
    bucket_names = []
    for bucket in bucket_data['Buckets']:
        bucket_names.append(bucket['Name'])

    return bucket_names

def list_bucket_objects(client, bucket_names):

    bucket_objects = {}

    for bucket in bucket_names:
        response = client.list_objects_v2(Bucket=bucket, MaxKeys=1000)
        if response.get('Contents'):
            bucket_objects[bucket] = response['Contents']
        else:
            bucket_objects[bucket] = []
            continue

        while response['IsTruncated']:
            reponse = client.list_objects_v2(Bucket=bucket, MaxKeys=1000, ContinuationToken=response['NextContinuationToken'])
            bucket_objects[bucket].extend(response['Contents'])

    return bucket_objects

def main(botoconfig, session):
    client = create_s3_client(botoconfig, session)

    print("\n[+] Starting Bucket Enumeration...\n")
    bucket_data = list_buckets(client)
    bucket_names = parse_bucket_data(bucket_data)

    for bucket in bucket_names:
        print("[+] Bucket Name: "+Fore.GREEN+"{}".format(bucket)+Style.RESET_ALL)

    print("\n[-] Do you want to enumerate objects in those buckets?")
    enumerate_objects = input("[-] WARNING: This could generate a lot of traffic [N/y]: ")

    if enumerate_objects.lower() == "y" or enumerate_objects.lower() == "yes":
        print("\n[+] Starting Bucket Objects Enumeration...")
        bucket_objects = list_bucket_objects(client, bucket_names)

        for bucket in bucket_names:
            print(Fore.GREEN + "\n[+] Objects in bucket: " + Style.RESET_ALL + "{}\n".format(bucket))
            for object in bucket_objects.get(bucket):
                print("- {}".format(object.get('Key')))

    return bucket_data