import bisect
import codecs
import collections as co
import csv
import io
import itertools as it
import math as m
import os
import shlex
import shutil
import time

try:
    import inotify_simple
except ModuleNotFoundError:
    inotify_simple = None

COLORS = [
    '1;34', # bold blue
    '1;31', # bold red
    '1;32', # bold green
    '1;35', # bold purple
    '1;33', # bold yellow
    '1;36', # bold cyan
    '34',   # blue
    '31',   # red
    '32',   # green
    '35',   # purple
    '33',   # yellow
    '36',   # cyan
]

CHARS_DOTS = " .':"
CHARS_BRAILLE = (
    '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴'
    '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶'
    '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼'
    '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾'
    '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵'
    '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷'
    '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽'
    '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿')
CHARS_POINTS_AND_LINES = 'o'

SI_PREFIXES = {
    18:  'E',
    15:  'P',
    12:  'T',
    9:   'G',
    6:   'M',
    3:   'K',
    0:   '',
    -3:  'm',
    -6:  'u',
    -9:  'n',
    -12: 'p',
    -15: 'f',
    -18: 'a',
}

SI2_PREFIXES = {
    60:  'Ei',
    50:  'Pi',
    40:  'Ti',
    30:  'Gi',
    20:  'Mi',
    10:  'Ki',
    0:   '',
    -10: 'mi',
    -20: 'ui',
    -30: 'ni',
    -40: 'pi',
    -50: 'fi',
    -60: 'ai',
}

# format a number to a strict character width using SI prefixes
def si(x, w=4):
    if x == 0:
        return '0'
    # figure out prefix and scale
    #
    # note we adjust this so that 100K = .1M, which has more info
    # per character
    p = 3*int(m.log(abs(x)*10, 10**3))
    p = min(18, max(-18, p))
    # format with enough digits
    s = '%.*f' % (w, abs(x) / (10.0**p))
    s = s.lstrip('0')
    # truncate but only digits that follow the dot
    if '.' in s:
        s = s[:max(s.find('.'), w-(2 if x < 0 else 1))]
        s = s.rstrip('0')
        s = s.rstrip('.')
    return '%s%s%s' % ('-' if x < 0 else '', s, SI_PREFIXES[p])

def si2(x, w=5):
    if x == 0:
        return '0'
    # figure out prefix and scale
    #
    # note we adjust this so that 128Ki = .1Mi, which has more info
    # per character
    p = 10*int(m.log(abs(x)*10, 2**10))
    p = min(30, max(-30, p))
    # format with enough digits
    s = '%.*f' % (w, abs(x) / (2.0**p))
    s = s.lstrip('0')
    # truncate but only digits that follow the dot
    if '.' in s:
        s = s[:max(s.find('.'), w-(3 if x < 0 else 2))]
        s = s.rstrip('0')
        s = s.rstrip('.')
    return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p])

# parse escape strings
def escape(s):
    return codecs.escape_decode(s.encode('utf8'))[0].decode('utf8')

def openio(path, mode='r', buffering=-1):
    # allow '-' for stdin/stdout
    if path == '-':
        if mode == 'r':
            return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
        else:
            return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
    else:
        return open(path, mode, buffering)

def inotifywait(paths):
    # wait for interesting events
    inotify = inotify_simple.INotify()
    flags = (inotify_simple.flags.ATTRIB
        | inotify_simple.flags.CREATE
        | inotify_simple.flags.DELETE
        | inotify_simple.flags.DELETE_SELF
        | inotify_simple.flags.MODIFY
        | inotify_simple.flags.MOVED_FROM
        | inotify_simple.flags.MOVED_TO
        | inotify_simple.flags.MOVE_SELF)

    # recurse into directories
    for path in paths:
        if os.path.isdir(path):
            for dir, _, files in os.walk(path):
                inotify.add_watch(dir, flags)
                for f in files:
                    inotify.add_watch(os.path.join(dir, f), flags)
        else:
            inotify.add_watch(path, flags)

    # wait for event
    inotify.read()

class LinesIO:
    def __init__(self, maxlen=None):
        self.maxlen = maxlen
        self.lines = co.deque(maxlen=maxlen)
        self.tail = io.StringIO()

        # trigger automatic sizing
        if maxlen == 0:
            self.resize(0)

    def write(self, s):
        # note using split here ensures the trailing string has no newline
        lines = s.split('\n')

        if len(lines) > 1 and self.tail.getvalue():
            self.tail.write(lines[0])
            lines[0] = self.tail.getvalue()
            self.tail = io.StringIO()

        self.lines.extend(lines[:-1])

        if lines[-1]:
            self.tail.write(lines[-1])

    def resize(self, maxlen):
        self.maxlen = maxlen
        if maxlen == 0:
            maxlen = shutil.get_terminal_size((80, 5))[1]
        if maxlen != self.lines.maxlen:
            self.lines = co.deque(self.lines, maxlen=maxlen)

    canvas_lines = 1
    def draw(self):
        # did terminal size change?
        if self.maxlen == 0:
            self.resize(0)

        # first thing first, give ourself a canvas
        while LinesIO.canvas_lines < len(self.lines):
            sys.stdout.write('\n')
            LinesIO.canvas_lines += 1

        # clear the bottom of the canvas if we shrink
        shrink = LinesIO.canvas_lines - len(self.lines)
        if shrink > 0:
            for i in range(shrink):
                sys.stdout.write('\r')
                if shrink-1-i > 0:
                    sys.stdout.write('\x1b[%dA' % (shrink-1-i))
                sys.stdout.write('\x1b[K')
                if shrink-1-i > 0:
                    sys.stdout.write('\x1b[%dB' % (shrink-1-i))
            sys.stdout.write('\x1b[%dA' % shrink)
            LinesIO.canvas_lines = len(self.lines)

        for i, line in enumerate(self.lines):
            # move cursor, clear line, disable/reenable line wrapping
            sys.stdout.write('\r')
            if len(self.lines)-1-i > 0:
                sys.stdout.write('\x1b[%dA' % (len(self.lines)-1-i))
            sys.stdout.write('\x1b[K')
            sys.stdout.write('\x1b[?7l')
            sys.stdout.write(line)
            sys.stdout.write('\x1b[?7h')
            if len(self.lines)-1-i > 0:
                sys.stdout.write('\x1b[%dB' % (len(self.lines)-1-i))
        sys.stdout.flush()

# parse different data representations
def dat(x):
    # allow the first part of an a/b fraction
    if '/' in x:
        x, _ = x.split('/', 1)

    # first try as int
    try:
        return int(x, 0)
    except ValueError:
        pass

    # then try as float
    try:
        return float(x)
        # just don't allow infinity or nan
        if m.isinf(x) or m.isnan(x):
            raise ValueError("invalid dat %r" % x)
    except ValueError:
        pass

    # else give up
    raise ValueError("invalid dat %r" % x)

# a hack log that preserves sign, with a linear region between -1 and 1
def symlog(x):
    if x > 1:
        return m.log(x)+1
    elif x < -1:
        return -m.log(-x)-1
    else:
        return x

class Plot:
    def __init__(self, width, height, *,
            xlim=None,
            ylim=None,
            xlog=False,
            ylog=False,
            braille=False,
            dots=False):
        # scale if we're printing with dots or braille
        self.width = 2*width if braille else width
        self.height = (4*height if braille
            else 2*height if dots
            else height)

        self.xlim = xlim or (0, width)
        self.ylim = ylim or (0, height)
        self.xlog = xlog
        self.ylog = ylog
        self.braille = braille
        self.dots = dots

        self.grid = [('',False)]*(self.width*self.height)

    def scale(self, x, y):
        # scale and clamp
        try:
            if self.xlog:
                x = int(self.width * (
                    (symlog(x)-symlog(self.xlim[0]))
                    / (symlog(self.xlim[1])-symlog(self.xlim[0]))))
            else:
                x = int(self.width * (
                    (x-self.xlim[0])
                    / (self.xlim[1]-self.xlim[0])))
            if self.ylog:
                y = int(self.height * (
                    (symlog(y)-symlog(self.ylim[0]))
                    / (symlog(self.ylim[1])-symlog(self.ylim[0]))))
            else:
                y = int(self.height * (
                    (y-self.ylim[0])
                    / (self.ylim[1]-self.ylim[0])))
        except ZeroDivisionError:
            x = 0
            y = 0
        return x, y

    def point(self, x, y, *,
            color=COLORS[0],
            char=True):
        # scale
        x, y = self.scale(x, y)

        # ignore out of bounds points
        if x >= 0 and x < self.width and y >= 0 and y < self.height:
            self.grid[x + y*self.width] = (color, char)

    def line(self, x1, y1, x2, y2, *,
            color=COLORS[0],
            char=True):
        # scale
        x1, y1 = self.scale(x1, y1)
        x2, y2 = self.scale(x2, y2)

        # incremental error line algorithm
        ex = abs(x2 - x1)
        ey = -abs(y2 - y1)
        dx = +1 if x1 < x2 else -1
        dy = +1 if y1 < y2 else -1
        e = ex + ey

        while True:
            if x1 >= 0 and x1 < self.width and y1 >= 0 and y1 < self.height:
                self.grid[x1 + y1*self.width] = (color, char)
            e2 = 2*e

            if x1 == x2 and y1 == y2:
                break

            if e2 > ey:
                e += ey
                x1 += dx

            if x1 == x2 and y1 == y2:
                break

            if e2 < ex:
                e += ex
                y1 += dy

        if x2 >= 0 and x2 < self.width and y2 >= 0 and y2 < self.height:
            self.grid[x2 + y2*self.width] = (color, char)

    def plot(self, coords, *,
            color=COLORS[0],
            char=True,
            line_char=True):
        # draw lines
        if line_char:
            for (x1, y1), (x2, y2) in zip(coords, coords[1:]):
                if y1 is not None and y2 is not None:
                    self.line(x1, y1, x2, y2,
                        color=color,
                        char=line_char)

        # draw points
        if char and (not line_char or char is not True):
            for x, y in coords:
                if y is not None:
                    self.point(x, y,
                        color=color,
                        char=char)

    def draw(self, row, *,
            color=False):
        # scale if needed
        if self.braille:
            xscale, yscale = 2, 4
        elif self.dots:
            xscale, yscale = 1, 2
        else:
            xscale, yscale = 1, 1

        y = self.height//yscale-1 - row
        row_ = []
        for x in range(self.width//xscale):
            best_f = ''
            best_c = False

            # encode into a byte
            b = 0
            for i in range(xscale*yscale):
                f, c = self.grid[x*xscale+(xscale-1-(i%xscale))
                        + (y*yscale+(i//xscale))*self.width]
                if c:
                    b |= 1 << i

                if f:
                    best_f = f
                if c and c is not True:
                    best_c = c

            # use byte to lookup character
            if b:
                if best_c:
                    c = best_c
                elif self.braille:
                    c = CHARS_BRAILLE[b]
                else:
                    c = CHARS_DOTS[b]
            else:
                c = ' '

            # color?
            if b and color and best_f:
                c = '\x1b[%sm%s\x1b[m' % (best_f, c)

            # draw axis in blank spaces
            if not b:
                if x == 0 and y == 0:
                    c = '+'
                elif x == 0 and y == self.height//yscale-1:
                    c = '^'
                elif x == self.width//xscale-1 and y == 0:
                    c = '>'
                elif x == 0:
                    c = '|'
                elif y == 0:
                    c = '-'

            row_.append(c)

        return ''.join(row_)

def collect(csv_paths, renames=[]):
    # collect results from CSV files
    results = []
    for path in csv_paths:
        try:
            with openio(path) as f:
                reader = csv.DictReader(f, restval='')
                for r in reader:
                    results.append(r)
        except FileNotFoundError:
            pass

    if renames:
        for r in results:
            # make a copy so renames can overlap
            r_ = {}
            for new_k, old_k in renames:
                if old_k in r:
                    r_[new_k] = r[old_k]
            r.update(r_)

    return results

def dataset(results, x=None, y=None, define=[]):
    # organize by 'by', x, and y
    dataset = {}
    i = 0
    for r in results:
        # filter results by matching defines
        if not all(k in r and r[k] in vs for k, vs in define):
            continue

        # find xs
        if x is not None:
            if x not in r:
                continue
            try:
                x_ = dat(r[x])
            except ValueError:
                continue
        else:
            x_ = i
            i += 1

        # find ys
        if y is not None:
            if y not in r:
                continue
            try:
                y_ = dat(r[y])
            except ValueError:
                continue
        else:
            y_ = None

        if y_ is not None:
            dataset[x_] = y_ + dataset.get(x_, 0)
        else:
            dataset[x_] = y_ or dataset.get(x_, None)

    return dataset

def datasets(results, by=None, x=None, y=None, define=[]):
    # filter results by matching defines
    results_ = []
    for r in results:
        if all(k in r and r[k] in vs for k, vs in define):
            results_.append(r)
    results = results_

    # if y not specified, try to guess from data
    if y is None:
        y = co.OrderedDict()
        for r in results:
            for k, v in r.items():
                if (by is None or k not in by) and v.strip():
                    try:
                        dat(v)
                        y[k] = True
                    except ValueError:
                        y[k] = False
        y = list(k for k,v in y.items() if v)

    if by is not None:
        # find all 'by' values
        ks = set()
        for r in results:
            ks.add(tuple(r.get(k, '') for k in by))
        ks = sorted(ks)

    # collect all datasets
    datasets = co.OrderedDict()
    for ks_ in (ks if by is not None else [()]):
        for x_ in (x if x is not None else [None]):
            for y_ in y:
                # hide x/y if there is only one field
                k_x = x_ if len(x or []) > 1 else ''
                k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else ''

                datasets[ks_ + (k_x, k_y)] = dataset(
                    results,
                    x_,
                    y_,
                    [(by_, {k_}) for by_, k_ in zip(by, ks_)]
                        if by is not None else [])

    return datasets

# some classes for organizing subplots into a grid
class Subplot:
    def __init__(self, **args):
        self.x = 0
        self.y = 0
        self.xspan = 1
        self.yspan = 1
        self.args = args

class Grid:
    def __init__(self, subplot, width=1.0, height=1.0):
        self.xweights = [width]
        self.yweights = [height]
        self.map = {(0,0): subplot}
        self.subplots = [subplot]

    def __repr__(self):
        return 'Grid(%r, %r)' % (self.xweights, self.yweights)

    @property
    def width(self):
        return len(self.xweights)

    @property
    def height(self):
        return len(self.yweights)

    def __iter__(self):
        return iter(self.subplots)

    def __getitem__(self, i):
        x, y = i
        if x < 0:
            x += len(self.xweights)
        if y < 0:
            y += len(self.yweights)

        return self.map[(x,y)]

    def merge(self, other, dir):
        if dir in ['above', 'below']:
            # first scale the two grids so they line up
            self_xweights = self.xweights
            other_xweights = other.xweights
            self_w = sum(self_xweights)
            other_w = sum(other_xweights)
            ratio = self_w / other_w
            other_xweights = [s*ratio for s in other_xweights]

            # now interleave xweights as needed
            new_xweights = []
            self_map = {}
            other_map = {}
            self_i = 0
            other_i = 0
            self_xweight = (self_xweights[self_i]
                if self_i < len(self_xweights) else m.inf)
            other_xweight = (other_xweights[other_i]
                if other_i < len(other_xweights) else m.inf)
            while self_i < len(self_xweights) and other_i < len(other_xweights):
                if other_xweight - self_xweight > 0.0000001:
                    new_xweights.append(self_xweight)
                    other_xweight -= self_xweight

                    new_i = len(new_xweights)-1
                    for j in range(len(self.yweights)):
                        self_map[(new_i, j)] = self.map[(self_i, j)]
                    for j in range(len(other.yweights)):
                        other_map[(new_i, j)] = other.map[(other_i, j)]
                    for s in other.subplots:
                        if s.x+s.xspan-1 == new_i:
                            s.xspan += 1
                        elif s.x > new_i:
                            s.x += 1

                    self_i += 1
                    self_xweight = (self_xweights[self_i]
                        if self_i < len(self_xweights) else m.inf)
                elif self_xweight - other_xweight > 0.0000001:
                    new_xweights.append(other_xweight)
                    self_xweight -= other_xweight

                    new_i = len(new_xweights)-1
                    for j in range(len(other.yweights)):
                        other_map[(new_i, j)] = other.map[(other_i, j)]
                    for j in range(len(self.yweights)):
                        self_map[(new_i, j)] = self.map[(self_i, j)]
                    for s in self.subplots:
                        if s.x+s.xspan-1 == new_i:
                            s.xspan += 1
                        elif s.x > new_i:
                            s.x += 1

                    other_i += 1
                    other_xweight = (other_xweights[other_i]
                        if other_i < len(other_xweights) else m.inf)
                else:
                    new_xweights.append(self_xweight)

                    new_i = len(new_xweights)-1
                    for j in range(len(self.yweights)):
                        self_map[(new_i, j)] = self.map[(self_i, j)]
                    for j in range(len(other.yweights)):
                        other_map[(new_i, j)] = other.map[(other_i, j)]

                    self_i += 1
                    self_xweight = (self_xweights[self_i]
                        if self_i < len(self_xweights) else m.inf)
                    other_i += 1
                    other_xweight = (other_xweights[other_i]
                        if other_i < len(other_xweights) else m.inf)

            # squish so ratios are preserved
            self_h = sum(self.yweights)
            other_h = sum(other.yweights)
            ratio = (self_h-other_h) / self_h
            self_yweights = [s*ratio for s in self.yweights]

            # finally concatenate the two grids
            if dir == 'above':
                for s in other.subplots:
                    s.y += len(self_yweights)
                self.subplots.extend(other.subplots)

                self.xweights = new_xweights
                self.yweights = self_yweights + other.yweights
                self.map = self_map | {(x, y+len(self_yweights)): s
                    for (x, y), s in other_map.items()}
            else:
                for s in self.subplots:
                    s.y += len(other.yweights)
                self.subplots.extend(other.subplots)

                self.xweights = new_xweights
                self.yweights = other.yweights + self_yweights
                self.map = other_map | {(x, y+len(other.yweights)): s
                    for (x, y), s in self_map.items()}

        if dir in ['right', 'left']:
            # first scale the two grids so they line up
            self_yweights = self.yweights
            other_yweights = other.yweights
            self_h = sum(self_yweights)
            other_h = sum(other_yweights)
            ratio = self_h / other_h
            other_yweights = [s*ratio for s in other_yweights]

            # now interleave yweights as needed
            new_yweights = []
            self_map = {}
            other_map = {}
            self_i = 0
            other_i = 0
            self_yweight = (self_yweights[self_i]
                if self_i < len(self_yweights) else m.inf)
            other_yweight = (other_yweights[other_i]
                if other_i < len(other_yweights) else m.inf)
            while self_i < len(self_yweights) and other_i < len(other_yweights):
                if other_yweight - self_yweight > 0.0000001:
                    new_yweights.append(self_yweight)
                    other_yweight -= self_yweight

                    new_i = len(new_yweights)-1
                    for j in range(len(self.xweights)):
                        self_map[(j, new_i)] = self.map[(j, self_i)]
                    for j in range(len(other.xweights)):
                        other_map[(j, new_i)] = other.map[(j, other_i)]
                    for s in other.subplots:
                        if s.y+s.yspan-1 == new_i:
                            s.yspan += 1
                        elif s.y > new_i:
                            s.y += 1

                    self_i += 1
                    self_yweight = (self_yweights[self_i]
                        if self_i < len(self_yweights) else m.inf)
                elif self_yweight - other_yweight > 0.0000001:
                    new_yweights.append(other_yweight)
                    self_yweight -= other_yweight

                    new_i = len(new_yweights)-1
                    for j in range(len(other.xweights)):
                        other_map[(j, new_i)] = other.map[(j, other_i)]
                    for j in range(len(self.xweights)):
                        self_map[(j, new_i)] = self.map[(j, self_i)]
                    for s in self.subplots:
                        if s.y+s.yspan-1 == new_i:
                            s.yspan += 1
                        elif s.y > new_i:
                            s.y += 1

                    other_i += 1
                    other_yweight = (other_yweights[other_i]
                        if other_i < len(other_yweights) else m.inf)
                else:
                    new_yweights.append(self_yweight)

                    new_i = len(new_yweights)-1
                    for j in range(len(self.xweights)):
                        self_map[(j, new_i)] = self.map[(j, self_i)]
                    for j in range(len(other.xweights)):
                        other_map[(j, new_i)] = other.map[(j, other_i)]

                    self_i += 1
                    self_yweight = (self_yweights[self_i]
                        if self_i < len(self_yweights) else m.inf)
                    other_i += 1
                    other_yweight = (other_yweights[other_i]
                        if other_i < len(other_yweights) else m.inf)

            # squish so ratios are preserved
            self_w = sum(self.xweights)
            other_w = sum(other.xweights)
            ratio = (self_w-other_w) / self_w
            self_xweights = [s*ratio for s in self.xweights]

            # finally concatenate the two grids
            if dir == 'right':
                for s in other.subplots:
                    s.x += len(self_xweights)
                self.subplots.extend(other.subplots)

                self.xweights = self_xweights + other.xweights
                self.yweights = new_yweights
                self.map = self_map | {(x+len(self_xweights), y): s
                    for (x, y), s in other_map.items()}
            else:
                for s in self.subplots:
                    s.x += len(other.xweights)
                self.subplots.extend(other.subplots)

                self.xweights = other.xweights + self_xweights
                self.yweights = new_yweights
                self.map = other_map | {(x+len(other.xweights), y): s
                    for (x, y), s in self_map.items()}

    def scale(self, width, height):
        self.xweights = [s*width for s in self.xweights]
        self.yweights = [s*height for s in self.yweights]

    @classmethod
    def fromargs(cls, width=1.0, height=1.0, *,
            subplots=[],
            **args):
        grid = cls(Subplot(**args))

        for dir, subargs in subplots:
            subgrid = cls.fromargs(
                width=subargs.pop('width',
                    0.5 if dir in ['right', 'left'] else width),
                height=subargs.pop('height',
                    0.5 if dir in ['above', 'below'] else height),
                **subargs)
            grid.merge(subgrid, dir)

        grid.scale(width, height)
        return grid

def main(csv_paths, *,
        by=None,
        x=None,
        y=None,
        define=[],
        color=False,
        braille=False,
        colors=None,
        chars=None,
        line_chars=None,
        points=False,
        points_and_lines=False,
        width=None,
        height=None,
        xlim=(None,None),
        ylim=(None,None),
        xlog=False,
        ylog=False,
        x2=False,
        y2=False,
        xunits='',
        yunits='',
        xlabel=None,
        ylabel=None,
        xticklabels=None,
        yticklabels=None,
        title=None,
        legend_right=False,
        legend_above=False,
        legend_below=False,
        subplot={},
        subplots=[],
        cat=False,
        keep_open=False,
        sleep=None,
        **args):
    # figure out what color should be
    if color == 'auto':
        color = sys.stdout.isatty()
    elif color == 'always':
        color = True
    else:
        color = False

    # what colors to use?
    if colors is not None:
        colors_ = colors
    else:
        colors_ = COLORS

    if chars is not None:
        chars_ = chars
    elif points_and_lines:
        chars_ = CHARS_POINTS_AND_LINES
    else:
        chars_ = [True]

    if line_chars is not None:
        line_chars_ = line_chars
    elif points_and_lines or not points:
        line_chars_ = [True]
    else:
        line_chars_ = [False]

    # allow escape codes in labels/titles
    title = escape(title).splitlines() if title is not None else []
    xlabel = escape(xlabel).splitlines() if xlabel is not None else []
    ylabel = escape(ylabel).splitlines() if ylabel is not None else []

    # separate out renames
    renames = list(it.chain.from_iterable(
        ((k, v) for v in vs)
        for k, vs in it.chain(by or [], x or [], y or [])))
    if by is not None:
        by = [k for k, _ in by]
    if x is not None:
        x = [k for k, _ in x]
    if y is not None:
        y = [k for k, _ in y]

    # create a grid of subplots
    grid = Grid.fromargs(
        subplots=subplots + subplot.pop('subplots', []),
        **subplot)

    for s in grid:
        # allow subplot params to override global params
        x2_ = s.args.get('x2', False) or x2
        y2_ = s.args.get('y2', False) or y2
        xunits_ = s.args.get('xunits', xunits)
        yunits_ = s.args.get('yunits', yunits)
        xticklabels_ = s.args.get('xticklabels', xticklabels)
        yticklabels_ = s.args.get('yticklabels', yticklabels)

        # label/titles are handled a bit differently in subplots
        subtitle = s.args.get('title')
        xsublabel = s.args.get('xlabel')
        ysublabel = s.args.get('ylabel')

        # allow escape codes in sublabels/subtitles
        subtitle = (escape(subtitle).splitlines()
            if subtitle is not None else [])
        xsublabel = (escape(xsublabel).splitlines()
            if xsublabel is not None else [])
        ysublabel = (escape(ysublabel).splitlines()
            if ysublabel is not None else [])

        # don't allow >2 ticklabels and render single ticklabels only once
        if xticklabels_ is not None:
            if len(xticklabels_) == 1:
                xticklabels_ = ["", xticklabels_[0]]
            elif len(xticklabels_) > 2:
                xticklabels_ = [xticklabels_[0], xticklabels_[-1]]
        if yticklabels_ is not None:
            if len(yticklabels_) == 1:
                yticklabels_ = ["", yticklabels_[0]]
            elif len(yticklabels_) > 2:
                yticklabels_ = [yticklabels_[0], yticklabels_[-1]]

        s.x2 = x2_
        s.y2 = y2_
        s.xunits = xunits_
        s.yunits = yunits_
        s.xticklabels = xticklabels_
        s.yticklabels = yticklabels_
        s.title = subtitle
        s.xlabel = xsublabel
        s.ylabel = ysublabel

    # preprocess margins so they can be shared
    for s in grid:
        s.xmargin = (
            len(s.ylabel) + (1 if s.ylabel else 0) # fit ysublabel
                + (1 if s.x > 0 else 0),           # space between
            ((5 if s.y2 else 4) + len(s.yunits)    # fit yticklabels
                if s.yticklabels is None
                else max((len(t) for t in s.yticklabels), default=0))
                + (1 if s.yticklabels != [] else 0),
        )
        s.ymargin = (
            len(s.xlabel),                   # fit xsublabel
            1 if s.xticklabels != [] else 0, # fit xticklabels
            len(s.title),                    # fit subtitle
        )

    for s in grid:
        # share margins so everything aligns nicely
        s.xmargin = (
            max(s_.xmargin[0] for s_ in grid if s_.x == s.x),
            max(s_.xmargin[1] for s_ in grid if s_.x == s.x),
        )
        s.ymargin = (
            max(s_.ymargin[0] for s_ in grid if s_.y == s.y),
            max(s_.ymargin[1] for s_ in grid if s_.y == s.y),
            max(s_.ymargin[-1] for s_ in grid if s_.y+s_.yspan == s.y+s.yspan),
        )

    def draw(f):
        def writeln(s=''):
            f.write(s)
            f.write('\n')
        f.writeln = writeln

        # first collect results from CSV files
        results = collect(csv_paths, renames)

        # then extract the requested datasets
        datasets_ = datasets(results, by, x, y, define)

        # figure out colors/chars here so that subplot defines
        # don't change them later, that'd be bad
        datacolors_ = {
            name: colors_[i % len(colors_)]
            for i, name in enumerate(datasets_.keys())}
        datachars_ = {
            name: chars_[i % len(chars_)]
            for i, name in enumerate(datasets_.keys())}
        dataline_chars_ = {
            name: line_chars_[i % len(line_chars_)]
            for i, name in enumerate(datasets_.keys())}

        # build legend?
        legend_width = 0
        if legend_right or legend_above or legend_below:
            legend_ = []
            for i, k in enumerate(datasets_.keys()):
                label = '%s%s' % (
                    '%s ' % chars_[i % len(chars_)]
                        if chars is not None
                        else '%s ' % line_chars_[i % len(line_chars_)]
                        if line_chars is not None
                        else '',
                    ','.join(k_ for k_ in k if k_))

                if label:
                    legend_.append(label)
                    legend_width = max(legend_width, len(label)+1)

        # figure out our canvas size
        if width is None:
            width_ = min(80, shutil.get_terminal_size((80, None))[0])
        elif width:
            width_ = width
        else:
            width_ = shutil.get_terminal_size((80, None))[0]

        if height is None:
            height_ = 17 + len(title) + len(xlabel)
        elif height:
            height_ = height
        else:
            height_ = shutil.get_terminal_size((None,
                17 + len(title) + len(xlabel)))[1]
            # make space for shell prompt
            if not keep_open:
                height_ -= 1

        # carve out space for the xlabel
        height_ -= len(xlabel)
        # carve out space for the ylabel
        width_ -= len(ylabel) + (1 if ylabel else 0)
        # carve out space for title
        height_ -= len(title)

        # carve out space for the legend
        if legend_right and legend_:
            width_ -= legend_width
        if legend_above and legend_:
            legend_cols = len(legend_)
            while True:
                legend_widths = [
                    max(len(l) for l in legend_[i::legend_cols])
                    for i in range(legend_cols)]
                if (legend_cols <= 1
                        or sum(legend_widths)+2*(legend_cols-1)
                            + max(sum(s.xmargin[:2]) for s in grid if s.x == 0)
                            <= width_):
                    break
                legend_cols -= 1
            height_ -= (len(legend_)+legend_cols-1) // legend_cols
        if legend_below and legend_:
            legend_cols = len(legend_)
            while True:
                legend_widths = [
                    max(len(l) for l in legend_[i::legend_cols])
                    for i in range(legend_cols)]
                if (legend_cols <= 1
                        or sum(legend_widths)+2*(legend_cols-1)
                            + max(sum(s.xmargin[:2]) for s in grid if s.x == 0)
                            <= width_):
                    break
                legend_cols -= 1
            height_ -= (len(legend_)+legend_cols-1) // legend_cols

        # figure out the grid dimensions
        #
        # note we floor to give the dimension tweaks the best chance of not
        # exceeding the requested dimensions, this means we usually are less
        # than the requested dimensions by quite a bit when we have many
        # subplots, but it's a tradeoff for a relatively simple implementation
        widths = [m.floor(w*width_) for w in grid.xweights]
        heights = [m.floor(w*height_) for w in grid.yweights]

        # tweak dimensions to allow all plots to have a minimum width,
        # this may force the plot to be larger than the requested dimensions,
        # but that's the best we can do
        for s in grid:
            # fit xunits
            minwidth = sum(s.xmargin) + max(2,
                2*((5 if s.x2 else 4)+len(s.xunits))
                    if s.xticklabels is None
                    else sum(len(t) for t in s.xticklabels))
            # fit yunits
            minheight = sum(s.ymargin) + 2

            i = 0
            while minwidth > sum(widths[s.x:s.x+s.xspan]):
                widths[s.x+i] += 1
                i = (i + 1) % s.xspan

            i = 0
            while minheight > sum(heights[s.y:s.y+s.yspan]):
                heights[s.y+i] += 1
                i = (i + 1) % s.yspan

        width_ = sum(widths)
        height_ = sum(heights)

        # create a plot for each subplot
        for s in grid:
            # allow subplot params to override global params
            define_ = define + s.args.get('define', [])
            xlim_ = s.args.get('xlim', xlim)
            ylim_ = s.args.get('ylim', ylim)
            xlog_ = s.args.get('xlog', False) or xlog
            ylog_ = s.args.get('ylog', False) or ylog

            # allow shortened ranges
            if len(xlim_) == 1:
                xlim_ = (0, xlim_[0])
            if len(ylim_) == 1:
                ylim_ = (0, ylim_[0])

            # data can be constrained by subplot-specific defines,
            # so re-extract for each plot
            subdatasets = datasets(results, by, x, y, define_)

            # find actual xlim/ylim
            xlim_ = (
                xlim_[0] if xlim_[0] is not None
                    else min(it.chain([0], (k
                        for r in subdatasets.values()
                        for k, v in r.items()
                        if v is not None))),
                xlim_[1] if xlim_[1] is not None
                    else max(it.chain([0], (k
                        for r in subdatasets.values()
                        for k, v in r.items()
                        if v is not None))))

            ylim_ = (
                ylim_[0] if ylim_[0] is not None
                    else min(it.chain([0], (v
                        for r in subdatasets.values()
                        for _, v in r.items()
                        if v is not None))),
                ylim_[1] if ylim_[1] is not None
                    else max(it.chain([0], (v
                        for r in subdatasets.values()
                        for _, v in r.items()
                        if v is not None))))

            # find actual width/height
            subwidth = sum(widths[s.x:s.x+s.xspan]) - sum(s.xmargin)
            subheight = sum(heights[s.y:s.y+s.yspan]) - sum(s.ymargin)

            # plot!
            plot = Plot(
                subwidth,
                subheight,
                xlim=xlim_,
                ylim=ylim_,
                xlog=xlog_,
                ylog=ylog_,
                braille=line_chars is None and braille,
                dots=line_chars is None and not braille)

            for name, dataset in subdatasets.items():
                plot.plot(
                    sorted((x,y) for x,y in dataset.items()),
                    color=datacolors_[name],
                    char=datachars_[name],
                    line_char=dataline_chars_[name])

            s.plot = plot
            s.width = subwidth
            s.height = subheight
            s.xlim = xlim_
            s.ylim = ylim_

        # now that everything's plotted, let's render things to the terminal

        # figure out margin
        xmargin = (
            len(ylabel) + (1 if ylabel else 0),
            sum(grid[0,0].xmargin[:2]),
        )
        ymargin = (
            sum(grid[0,0].ymargin[:2]),
            grid[-1,-1].ymargin[-1],
        )

        # draw title?
        for line in title:
            f.writeln('%*s%s' % (
                sum(xmargin[:2]), '',
                line.center(width_-xmargin[1])))

        # draw legend_above?
        if legend_above and legend_:
            for i in range(0, len(legend_), legend_cols):
                f.writeln('%*s%s' % (
                    max(sum(xmargin[:2])
                        + (width_-xmargin[1]
                            - (sum(legend_widths)+2*(legend_cols-1)))
                            // 2,
                        0), '',
                    '  '.join('%s%s%s' % (
                        '\x1b[%sm' % colors_[(i+j) % len(colors_)]
                            if color else '',
                        '%-*s' % (legend_widths[j], legend_[i+j]),
                        '\x1b[m'
                            if color else '')
                        for j in range(min(legend_cols, len(legend_)-i)))))

        for row in range(height_):
            # draw ylabel?
            f.write(
                '%s ' % ''.join(
                    ('%*s%s%*s' % (
                        ymargin[-1], '',
                        line.center(height_-sum(ymargin)),
                        ymargin[0], ''))[row]
                    for line in ylabel)
                if ylabel else '')

            for x_ in range(grid.width):
                # figure out the grid x/y position
                subrow = row
                y_ = len(heights)-1
                while subrow >= heights[y_]:
                    subrow -= heights[y_]
                    y_ -= 1

                s = grid[x_, y_]
                subrow = row - sum(heights[s.y+s.yspan:])

                # header
                if subrow < s.ymargin[-1]:
                    # draw subtitle?
                    if subrow < len(s.title):
                        f.write('%*s%s' % (
                            sum(s.xmargin[:2]), '',
                            s.title[subrow].center(s.width)))
                    else:
                        f.write('%*s%*s' % (
                            sum(s.xmargin[:2]), '',
                            s.width, ''))
                # draw plot?
                elif subrow-s.ymargin[-1] < s.height:
                    subrow = subrow-s.ymargin[-1]

                    # draw ysublabel?
                    f.write('%-*s' % (
                        s.xmargin[0],
                        '%s ' % ''.join(
                                line.center(s.height)[subrow]
                                for line in s.ylabel)
                            if s.ylabel else ''))

                    # draw yunits?
                    if subrow == 0 and s.yticklabels != []:
                        f.write('%*s' % (
                            s.xmargin[1],
                            ((si2 if s.y2 else si)(s.ylim[1]) + s.yunits
                                if s.yticklabels is None
                                else s.yticklabels[1])
                                + ' '))
                    elif subrow == s.height-1 and s.yticklabels != []:
                        f.write('%*s' % (
                            s.xmargin[1],
                            ((si2 if s.y2 else si)(s.ylim[0]) + s.yunits
                                if s.yticklabels is None
                                else s.yticklabels[0])
                                + ' '))
                    else:
                        f.write('%*s' % (
                            s.xmargin[1], ''))

                    # draw plot!
                    f.write(s.plot.draw(subrow, color=color))

                # footer
                else:
                    subrow = subrow-s.ymargin[-1]-s.height

                    # draw xunits?
                    if subrow < (1 if s.xticklabels != [] else 0):
                        f.write('%*s%-*s%*s%*s' % (
                            sum(s.xmargin[:2]), '',
                            (5 if s.x2 else 4) + len(s.xunits)
                                if s.xticklabels is None
                                else len(s.xticklabels[0]),
                            (si2 if s.x2 else si)(s.xlim[0]) + s.xunits
                                if s.xticklabels is None
                                else s.xticklabels[0],
                            s.width - (2*((5 if s.x2 else 4)+len(s.xunits))
                                if s.xticklabels is None
                                else sum(len(t) for t in s.xticklabels)), '',
                            (5 if s.x2 else 4) + len(s.xunits)
                                if s.xticklabels is None
                                else len(s.xticklabels[1]),
                            (si2 if s.x2 else si)(s.xlim[1]) + s.xunits
                                if s.xticklabels is None
                                else s.xticklabels[1]))
                    # draw xsublabel?
                    elif (subrow < s.ymargin[1]
                            or subrow-s.ymargin[1] >= len(s.xlabel)):
                        f.write('%*s%*s' % (
                            sum(s.xmargin[:2]), '',
                            s.width, ''))
                    else:
                        f.write('%*s%s' % (
                            sum(s.xmargin[:2]), '',
                            s.xlabel[subrow-s.ymargin[1]].center(s.width)))

            # draw legend_right?
            if (legend_right and legend_
                    and row >= ymargin[-1]
                    and row-ymargin[-1] < len(legend_)):
                j = row-ymargin[-1]
                f.write(' %s%s%s' % (
                    '\x1b[%sm' % colors_[j % len(colors_)] if color else '',
                    legend_[j],
                    '\x1b[m' if color else ''))

            f.writeln()

        # draw xlabel?
        for line in xlabel:
            f.writeln('%*s%s' % (
                sum(xmargin[:2]), '',
                line.center(width_-xmargin[1])))

        # draw legend below?
        if legend_below and legend_:
            for i in range(0, len(legend_), legend_cols):
                f.writeln('%*s%s' % (
                    max(sum(xmargin[:2])
                        + (width_-xmargin[1]
                            - (sum(legend_widths)+2*(legend_cols-1)))
                            // 2,
                        0), '',
                    '  '.join('%s%s%s' % (
                        '\x1b[%sm' % colors_[(i+j) % len(colors_)]
                            if color else '',
                        '%-*s' % (legend_widths[j], legend_[i+j]),
                        '\x1b[m'
                            if color else '')
                        for j in range(min(legend_cols, len(legend_)-i)))))

    if keep_open:
        try:
            while True:
                if cat:
                    draw(sys.stdout)
                else:
                    ring = LinesIO()
                    draw(ring)
                    ring.draw()

                # try to inotifywait
                if inotify_simple is not None:
                    ptime = time.time()
                    inotifywait(csv_paths)
                    # sleep for a minimum amount of time, this helps issues
                    # around rapidly updating files
                    time.sleep(max(0, (sleep or 0.01) - (time.time()-ptime)))
                else:
                    time.sleep(sleep or 0.1)
        except KeyboardInterrupt:
            pass

        if cat:
            draw(sys.stdout)
        else:
            ring = LinesIO()
            draw(ring)
            ring.draw()
        sys.stdout.write('\n')
    else:
        draw(sys.stdout)

if __name__ == "__main__":
    import sys
    import argparse
    parser = argparse.ArgumentParser(
        description="Plot CSV files in terminal.",
        allow_abbrev=False)
    parser.add_argument(
        'csv_paths',
        nargs='*',
        help="Input *.csv files.")
    parser.add_argument(
        '-b', '--by',
        action='append',
        type=lambda x: (
            lambda k,v=None: (k, v.split(',') if v is not None else ())
            )(*x.split('=', 1)),
        help="Group by this field. Can rename fields with new_name=old_name.")
    parser.add_argument(
        '-x',
        action='append',
        type=lambda x: (
            lambda k,v=None: (k, v.split(',') if v is not None else ())
            )(*x.split('=', 1)),
        help="Field to use for the x-axis. Can rename fields with "
            "new_name=old_name.")
    parser.add_argument(
        '-y',
        action='append',
        type=lambda x: (
            lambda k,v=None: (k, v.split(',') if v is not None else ())
            )(*x.split('=', 1)),
        help="Field to use for the y-axis. Can rename fields with "
            "new_name=old_name.")
    parser.add_argument(
        '-D', '--define',
        type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)),
        action='append',
        help="Only include results where this field is this value. May include "
            "comma-separated options.")
    parser.add_argument(
        '--color',
        choices=['never', 'always', 'auto'],
        default='auto',
        help="When to use terminal colors. Defaults to 'auto'.")
    parser.add_argument(
        '-⣿', '--braille',
        action='store_true',
        help="Use 2x4 unicode braille characters. Note that braille characters "
            "sometimes suffer from inconsistent widths.")
    parser.add_argument(
        '-.', '--points',
        action='store_true',
        help="Only draw data points.")
    parser.add_argument(
        '-!', '--points-and-lines',
        action='store_true',
        help="Draw data points and lines.")
    parser.add_argument(
        '--colors',
        type=lambda x: [x.strip() for x in x.split(',')],
        help="Comma-separated colors to use.")
    parser.add_argument(
        '--chars',
        help="Characters to use for points.")
    parser.add_argument(
        '--line-chars',
        help="Characters to use for lines.")
    parser.add_argument(
        '-W', '--width',
        nargs='?',
        type=lambda x: int(x, 0),
        const=0,
        help="Width in columns. 0 uses the terminal width. Defaults to "
            "min(terminal, 80).")
    parser.add_argument(
        '-H', '--height',
        nargs='?',
        type=lambda x: int(x, 0),
        const=0,
        help="Height in rows. 0 uses the terminal height. Defaults to 17.")
    parser.add_argument(
        '-X', '--xlim',
        type=lambda x: tuple(
            dat(x) if x.strip() else None
            for x in x.split(',')),
        help="Range for the x-axis.")
    parser.add_argument(
        '-Y', '--ylim',
        type=lambda x: tuple(
            dat(x) if x.strip() else None
            for x in x.split(',')),
        help="Range for the y-axis.")
    parser.add_argument(
        '--xlog',
        action='store_true',
        help="Use a logarithmic x-axis.")
    parser.add_argument(
        '--ylog',
        action='store_true',
        help="Use a logarithmic y-axis.")
    parser.add_argument(
        '--x2',
        action='store_true',
        help="Use base-2 prefixes for the x-axis.")
    parser.add_argument(
        '--y2',
        action='store_true',
        help="Use base-2 prefixes for the y-axis.")
    parser.add_argument(
        '--xunits',
        help="Units for the x-axis.")
    parser.add_argument(
        '--yunits',
        help="Units for the y-axis.")
    parser.add_argument(
        '--xlabel',
        help="Add a label to the x-axis.")
    parser.add_argument(
        '--ylabel',
        help="Add a label to the y-axis.")
    parser.add_argument(
        '--xticklabels',
        type=lambda x:
            [x.strip() for x in x.split(',')]
            if x.strip() else [],
        help="Comma separated xticklabels.")
    parser.add_argument(
        '--yticklabels',
        type=lambda x:
            [x.strip() for x in x.split(',')]
            if x.strip() else [],
        help="Comma separated yticklabels.")
    parser.add_argument(
        '-t', '--title',
        help="Add a title.")
    parser.add_argument(
        '-l', '--legend-right',
        action='store_true',
        help="Place a legend to the right.")
    parser.add_argument(
        '--legend-above',
        action='store_true',
        help="Place a legend above.")
    parser.add_argument(
        '--legend-below',
        action='store_true',
        help="Place a legend below.")
    class AppendSubplot(argparse.Action):
        @staticmethod
        def parse(value):
            import copy
            subparser = copy.deepcopy(parser)
            next(a for a in subparser._actions
                if '--width' in a.option_strings).type = float
            next(a for a in subparser._actions
                if '--height' in a.option_strings).type = float
            return subparser.parse_intermixed_args(shlex.split(value or ""))
        def __call__(self, parser, namespace, value, option):
            if not hasattr(namespace, 'subplots'):
                namespace.subplots = []
            namespace.subplots.append((
                option.split('-')[-1],
                self.__class__.parse(value)))
    parser.add_argument(
        '--subplot-above',
        action=AppendSubplot,
        help="Add subplot above with the same dataset. Takes an arg string to "
            "control the subplot which supports most (but not all) of the "
            "parameters listed here. The relative dimensions of the subplot "
            "can be controlled with -W/-H which now take a percentage.")
    parser.add_argument(
        '--subplot-below',
        action=AppendSubplot,
        help="Add subplot below with the same dataset.")
    parser.add_argument(
        '--subplot-left',
        action=AppendSubplot,
        help="Add subplot left with the same dataset.")
    parser.add_argument(
        '--subplot-right',
        action=AppendSubplot,
        help="Add subplot right with the same dataset.")
    parser.add_argument(
        '--subplot',
        type=AppendSubplot.parse,
        help="Add subplot-specific arguments to the main plot.")
    parser.add_argument(
        '-z', '--cat',
        action='store_true',
        help="Pipe directly to stdout.")
    parser.add_argument(
        '-k', '--keep-open',
        action='store_true',
        help="Continue to open and redraw the CSV files in a loop.")
    parser.add_argument(
        '-s', '--sleep',
        type=float,
        help="Time in seconds to sleep between redraws when running with -k. "
            "Defaults to 0.01.")

    def dictify(ns):
        if hasattr(ns, 'subplots'):
            ns.subplots = [(dir, dictify(subplot_ns))
                for dir, subplot_ns in ns.subplots]
        if ns.subplot is not None:
            ns.subplot = dictify(ns.subplot)
        return {k: v
            for k, v in vars(ns).items()
            if v is not None}

    sys.exit(main(**dictify(parser.parse_intermixed_args())))


from collections import ChainMap
import math

from natsort import natsorted
import numpy as np
import pandas as pd
from scipy import spatial

from libtiger.hunter.isotope_score import IsotopeHunter
from libtiger.reader import mzml_spectra_reader

def get_star_isotope_score(score_summary: dict, isotope_score: float = 75.0):
    score_summary["isotope_score"] = round(isotope_score, 6)

    return score_summary

def get_rank_obs_peaks(ms2_df: pd.DataFrame, res_space_info: dict):
    rank_obs_dct = {}
    for sub_res in ["class", "frag", "nl"]:
        residue_df = res_space_info.get(f"{sub_res}_space", pd.DataFrame())
        if isinstance(residue_df, pd.DataFrame) and not residue_df.empty:
            residue_range = res_space_info.get(f"{sub_res}_mz_range", {})
            res_mz_min = residue_range.get("min_mz", 0)
            res_mz_max = residue_range.get("max_mz", 0)
            if res_mz_min and res_mz_max:
                res_range_ms2_df = ms2_df[
                    (ms2_df["mz"] >= res_mz_min)
                    & (ms2_df["mz"] <= res_mz_max)
                ].copy()
            else:
                res_range_ms2_df = ms2_df.copy()
            res_range_ms2_df.sort_values(by="i", ascending=False, inplace=True)
            res_range_ms2_df.reset_index(drop=True, inplace=True)
            res_range_ms2_pkl = list(zip(res_range_ms2_df["mz"], res_range_ms2_df["i"]))
            sub_obs_dct = {}
            rank_counter = 0
            for mz_tp in res_range_ms2_pkl:
                peak_mz = mz_tp[0]
                peak_i = mz_tp[1]
                sub_obs_df = residue_df[
                    (residue_df["min_mz"] <= peak_mz) & (residue_df["max_mz"] >= peak_mz)
                ].copy()
                if not sub_obs_df.empty:
                    sub_obs_df["ppm"] = (
                        1000000 * (peak_mz - sub_obs_df["mz"]) / sub_obs_df["mz"]
                    )
                    sub_obs_df["abs_ppm"] = sub_obs_df["ppm"].abs()
                    sub_obs_df[["name", "adduct"]] = sub_obs_df["label"].str.split(
                        "#", 1, expand=True
                    )
                    if rank_counter < 10:
                        sub_obs_dct[rank_counter] = {
                            "rank": rank_counter,
                            "obs_mz": peak_mz,
                            "obs_i": peak_i,
                            "obs_df": sub_obs_df,
                        }
                        # if sub_obs_df.shape[0] == 1:
                        #     theo_mz = sub_obs_df["mz"].values.tolist()[0]
                        #     label = sub_obs_df["label"].values.tolist()[0]
                        #     ppm = 1000000 * (peak_mz - theo_mz) / theo_mz
                        #     abs_ppm = abs(ppm)
                        # else:
                        #     sub_obs_df.at[:, "ppm"] = (
                        #         1000000 * (peak_mz - sub_obs_df["mz"]) / sub_obs_df["mz"]
                        #     )
                        #     sub_obs_df.at[:, "abs_ppm"] = sub_obs_df["ppm"].abs()
                        #     sub_obs_df.sort_values(by="abs_ppm", ascending=True, inplace=True)
                        #     sub_obs_df.reset_index(drop=True, inplace=True)
                        #     theo_mz = sub_obs_df["mz"].values.tolist()[0]
                        #     label = sub_obs_df["label"].values.tolist()[0]
                        #     ppm = sub_obs_df["ppm"].values.tolist()[0]
                        #     abs_ppm = sub_obs_df["abs_ppm"].values.tolist()[0]
                        # if "#" in label:
                        #     lipid_name, adduct = label.split("#")
                        #     raw_sub_obs_dct[rank_counter] = {
                        #         "rank": rank_counter,
                        #         "mz": peak_mz,
                        #         "i": peak_i,
                        #         "theo_mz": theo_mz,
                        #         "ppm": ppm,
                        #         "abs_ppm": abs_ppm,
                        #         "label": label,
                        #         "name": lipid_name,
                        #         "adduct": adduct,
                        #     }
                        rank_counter += 1
                    else:
                        pass
            # if raw_sub_obs_dct:
            #     sub_obs_peaks_df = pd.DataFrame(data=raw_sub_obs_dct).T
            #     sub_obs_peaks_df.sort_values(
            #         by=["abs_ppm", "i"], ascending=[True, False], inplace=True
            #     )
            #     sub_obs_peaks_df.drop_duplicates(["label"], keep="first", inplace=True)
            #     sub_obs_peaks_df["rank"] = sub_obs_peaks_df["rank"].astype(int)
            #     sub_obs_peaks_df.set_index("rank", drop=True, inplace=True)
            #     sub_obs_dct = sub_obs_peaks_df.to_dict(orient="index")
            # else:
            #     sub_obs_dct = {}

            rank_obs_dct[sub_res] = sub_obs_dct

    return rank_obs_dct

def get_msp_vectors(msp_info: dict, mz_power: float = 3, i_power: float = 0.6):
    msp_vectors = []
    peak_pairs = list(zip(msp_info["mz"], msp_info["i"]))
    for pk_pair in peak_pairs:
        msp_vectors.append((pk_pair[0] ** mz_power) * (pk_pair[1] ** i_power))
    return msp_vectors

def hunt_isotope_score(
    r,
    neg_pr_df,
    pos_pr_df,
    neg_residue_space,
    pos_residue_space,
    selection_window,
    s_info_df,
    sum_spec_dct,
    ms1_ppm,
    ms1_abs_threshold,
    min_isotope_score,
):
    sum_isotope_score_info_dct = {}
    isotope_hunter = IsotopeHunter()
    ms2_dda_event_idx = r["dda_event_idx"]
    ms2_spec_index = r["spec_index"]
    ms2_pr_mz = r["MS2_PR_mz"]
    ms2_polarity = r["polarity"]

    # Due to the selection window for MS2 (usually 1 or 1.5 --> +/- 0.5 or +/- 0.75)
    # Precursor m/z in mzML MS2 can have large differences than actual MS1 precursor m/z
    if ms2_polarity == "neg":
        pr_candidates_df = neg_pr_df[
            (neg_pr_df["min_mz"] >= ms2_pr_mz - selection_window)
            & (neg_pr_df["min_mz"] <= ms2_pr_mz + selection_window)
            & (neg_pr_df["max_mz"] >= ms2_pr_mz - selection_window)
            & (neg_pr_df["max_mz"] <= ms2_pr_mz + selection_window)
        ].copy()
        pr_residue_space = neg_residue_space

    elif ms2_polarity == "pos":
        pr_candidates_df = pos_pr_df[
            (pos_pr_df["min_mz"] <= ms2_pr_mz + selection_window)
            & (pos_pr_df["max_mz"] >= ms2_pr_mz - selection_window)
        ].copy()
        pr_residue_space = pos_residue_space
    else:
        pr_candidates_df = pd.DataFrame()
        pr_residue_space = {}

    if not pr_candidates_df.empty:
        pr_ms1_df = s_info_df[
            (s_info_df["dda_event_idx"] == ms2_dda_event_idx)
            & (s_info_df["DDA_rank"] == 0)
            & (s_info_df["spec_index"] <= ms2_spec_index)
            & (s_info_df["polarity"] == ms2_polarity)
        ].copy()
        if not pr_ms1_df.empty:
            pr_ms1_spec_idx = sorted(pr_ms1_df["spec_index"].tolist())[0]
            pr_ms1_spec_df = sum_spec_dct.get(pr_ms1_spec_idx)
            pr_ms1_spec_df.sort_values(by="mz", ascending=True, inplace=True)
            unique_formula_lst = list(
                set(
                    zip(
                        pr_candidates_df["formula"].tolist(),
                        pr_candidates_df["mz"].tolist(),
                    )
                )
            )
            obs_ms1_pr_lst = []
            for chg_pr_tp in unique_formula_lst:
                chg_pr_formula = chg_pr_tp[0]
                (
                    ms1_obs_pr_mz,
                    ms1_obs_pr_i,
                    ms1_obs_pr_ppm,
                ) = mzml_spectra_reader.find_ms1_pr(
                    ms1_df=pr_ms1_spec_df,
                    mz_lib=chg_pr_tp[1],
                    ms1_ppm=ms1_ppm,
                )
                obs_ms1_pr_lst.append(
                    [
                        chg_pr_formula,
                        chg_pr_tp[1],
                        ms1_obs_pr_mz,
                        ms1_obs_pr_i,
                        ms1_obs_pr_ppm,
                        abs(ms1_obs_pr_ppm),
                    ]
                )
            raw_obs_ms1_pr_df = pd.DataFrame(
                obs_ms1_pr_lst,
                columns=["chg_pr_formula", "lib_mz", "obs_mz", "i", "ppm", "abs_ppm"],
            )
            obs_ms1_pr_df = raw_obs_ms1_pr_df[
                raw_obs_ms1_pr_df["abs_ppm"] <= ms1_ppm
            ].copy()
            obs_ms1_pr_df.sort_values(
                by=["i", "abs_ppm"], ascending=[False, True], inplace=True
            )
            obs_ms1_pr_df.reset_index(drop=True, inplace=True)
            obs_ms1_pr_df.drop_duplicates(keep="first", inplace=True)
            top_obs_ms1_pr_df = obs_ms1_pr_df.head(3)
            if not top_obs_ms1_pr_df.empty:
                top_obs_ms1_pr_dct = top_obs_ms1_pr_df.to_dict(orient="index")
            else:
                top_obs_ms1_pr_dct = {}
            for obs_ms1_pr_idx in top_obs_ms1_pr_dct:
                chg_pr_formula = top_obs_ms1_pr_dct[obs_ms1_pr_idx].get(
                    "chg_pr_formula"
                )
                ms1_obs_pr_mz = top_obs_ms1_pr_dct[obs_ms1_pr_idx].get("obs_mz", 0)
                isotope_score_info_dct = isotope_hunter.get_isotope_score(
                    ms1_obs_pr_mz,
                    top_obs_ms1_pr_dct[obs_ms1_pr_idx].get("i"),
                    chg_pr_formula,
                    pr_ms1_spec_df,
                    isotope_number=2,
                )
                isotope_score = isotope_score_info_dct.get("isotope_score", 0)
                observed_i = top_obs_ms1_pr_dct[obs_ms1_pr_idx].get("i", 0)
                observed_ppm = top_obs_ms1_pr_dct[obs_ms1_pr_idx].get("ppm", 999)
                print(isotope_score, observed_ppm, observed_i)
                if (
                    observed_i >= ms1_abs_threshold
                    and isotope_score >= min_isotope_score
                    and abs(observed_ppm) <= ms1_ppm
                ):

                    isotope_score_info_dct["charged_formula"] = chg_pr_formula
                    isotope_score_info_dct["pr_residue_space"] = pr_residue_space
                    isotope_score_info_dct["mz"] = top_obs_ms1_pr_dct[
                        obs_ms1_pr_idx
                    ].get("lib_mz", 0)
                    isotope_score_info_dct["observed_mz"] = ms1_obs_pr_mz
                    isotope_score_info_dct["observed_i"] = observed_i
                    isotope_score_info_dct["observed_ppm"] = observed_ppm
                    isotope_score_info_dct["ms1_scan_id"] = pr_ms1_spec_idx
                    isotope_score_info_dct["ms1_scan_time"] = pr_ms1_spec_df["rt"].max()
                    isotope_score_info_dct["ms2_polarity"] = pr_ms1_spec_df["rt"].max()
                    isotope_score_info_dct["ms1_df"] = pr_ms1_spec_df.drop(columns="rt")
                    isotope_score_info_dct["dda_event_id"] = ms2_dda_event_idx
                    isotope_score_info_dct["ms2_scan_id"] = ms2_spec_index
                    isotope_score_info_dct["ms2_scan_time"] = r["scan_time"]
                    isotope_score_info_dct["ms2_polarity"] = ms2_polarity
                    isotope_score_info_dct["ms2_precursor_mz"] = ms2_pr_mz
                    isotope_score_info_dct["ms2_df"] = sum_spec_dct.get(ms2_spec_index)

                    print(
                        f"isotope score check passed: {isotope_score} for {chg_pr_formula} @ {ms1_obs_pr_mz}"
                    )
                    sum_isotope_score_info_dct[
                        f"{chg_pr_formula}@{ms2_spec_index}"
                    ] = isotope_score_info_dct

    return sum_isotope_score_info_dct

def hunt_rank_score(
    pr_candidates: dict,
    ms2_df: pd.DataFrame,
    res_space_info: dict,
    weight_df: pd.DataFrame,
    rank_score_threshold: int = 30,
):
    rank_score_results = {}
    rank_obs_dct = get_rank_obs_peaks(
        ms2_df,
        res_space_info,
    )

    for pr_candidate in pr_candidates:
        pr_candidate_info = pr_candidates[pr_candidate]
        pr_name = pr_candidate_info.get("name", "")
        pr_info = pr_candidate_info.get("info", {})
        pr_rank_score = 0

        signal_labels = {}
        noise_labels = {}
        observed_fragments = {}
        observed_neutral_losses = {}
        observed_specific_peaks = {}
        observed_residues = []
        obs_matches_df = pd.DataFrame()
        res_name_lst = []
        for res_idx in ["FA1", "FA2", "FA3"]:
            res_name = pr_info.get(res_idx, {}).get("name", None)
            if res_name:
                res_name_lst.append(res_name)
        unique_res = len(list(set(res_name_lst)))
        frag_label_dct = pr_candidate_info.get("ions", {}).get("frag_info", {})
        nl_label_dct = pr_candidate_info.get("ions", {}).get("nl_info", {})
        class_label_dct = pr_candidate_info.get("ions", {}).get("class_info", {})
        nl_name_lst = list(set([nl_lb.split("#")[0] for nl_lb in nl_label_dct]))

        for sub_res in ["frag", "nl"]:
            if sub_res in rank_obs_dct:
                sub_rank_obs_dct = rank_obs_dct.get(sub_res, {})
                for rank in sub_rank_obs_dct:
                    is_count_for_score = False

                    if rank < 10:
                        obs_mz = sub_rank_obs_dct[rank].get("obs_mz", 0)
                        obs_i = sub_rank_obs_dct[rank].get("obs_i", 0)
                        obs_matches_df = sub_rank_obs_dct[rank].get("obs_df", pd.DataFrame)
                        obs_peak = [obs_mz, obs_i]
                        # obs_label = sub_rank_obs_dct[rank].get("label")
                        # obs_name = sub_rank_obs_dct[rank].get("name")
                        # obs_ppm = round(sub_rank_obs_dct[rank].get("ppm"), 1)
                        # obs_peak = [
                        #     sub_rank_obs_dct[rank].get("mz"),
                        #     sub_rank_obs_dct[rank].get("i"),
                        # ]

                        if obs_matches_df.shape[0] > 1:
                            if sub_res == "frag":
                                fit_obs_res_df = obs_matches_df[
                                    obs_matches_df["name"].isin(res_name_lst)
                                ].copy()
                            elif sub_res == "nl":
                                fit_obs_res_df = obs_matches_df[
                                    obs_matches_df["name"].isin(nl_name_lst)
                                ].copy()
                            else:
                                fit_obs_res_df = pd.DataFrame()
                            if fit_obs_res_df.shape[0] == 1:
                                obs_res_df = fit_obs_res_df
                            elif fit_obs_res_df.shape[0] > 1:
                                fit_obs_res_df.sort_values(
                                    by="abs_ppm", ascending=True, inplace=True
                                )
                                obs_res_df = fit_obs_res_df.head(1)
                            else:
                                obs_res_df = obs_matches_df.head(1)
                        else:
                            obs_res_df = obs_matches_df.copy()
                        obs_res_df.reset_index(drop=True, inplace=True)
                        obs_res_dct = obs_res_df.to_dict(orient="index")
                        obs_label = obs_res_dct[0].get("label")
                        obs_name = obs_res_dct[0].get("name")
                        obs_ppm = round(obs_res_dct[0].get("ppm"), 1)

                        if sub_res == "frag":
                            if obs_name in res_name_lst:
                                if unique_res >= 1 and obs_name not in observed_residues:
                                    is_count_for_score = True
                                elif unique_res == 1 and res_name in observed_residues:
                                    is_count_for_score = True
                                else:
                                    pass
                                if is_count_for_score:
                                    pr_rank_score += 40 * (1 - rank/10)
                                    observed_fragments[obs_label] = {
                                        "ppm": obs_ppm,
                                        "rank": rank,
                                    }
                                    signal_labels[obs_label] = obs_peak
                                    observed_residues.append(obs_name)
                                else:
                                    if obs_label not in frag_label_dct:
                                        noise_labels[obs_label] = obs_peak
                                    elif (
                                        obs_label in frag_label_dct
                                        and obs_label not in signal_labels
                                    ):
                                        signal_labels[obs_label] = obs_peak
                                        observed_neutral_losses[obs_label] = {
                                            "ppm": obs_ppm,
                                            "rank": rank,
                                        }
                                    else:
                                        pass
                            else:
                                if obs_label not in frag_label_dct:
                                    noise_labels[obs_label] = obs_peak

                        elif sub_res == "nl":
                            if (
                                obs_name in nl_name_lst
                                and unique_res > 1
                                and obs_name not in observed_residues
                            ):
                                is_count_for_score = True
                            elif (
                                obs_name in nl_name_lst
                                and unique_res == 1
                                and obs_name in observed_residues
                            ):
                                is_count_for_score = True
                            else:
                                pass
                            if is_count_for_score:
                                pr_rank_score += 10 * (10 - rank) * 0.1
                                observed_neutral_losses[obs_label] = {
                                    "ppm": obs_ppm,
                                    "rank": rank,
                                }
                                signal_labels[obs_label] = obs_peak
                                observed_residues.append(obs_name)
                            else:
                                if obs_label not in nl_label_dct:
                                    noise_labels[obs_label] = obs_peak
                                elif (
                                    obs_label in nl_label_dct
                                    and obs_label not in signal_labels
                                ):
                                    signal_labels[obs_label] = obs_peak
                                    observed_neutral_losses[obs_label] = {
                                        "ppm": obs_ppm,
                                        "rank": rank,
                                    }
                                else:
                                    pass
                        else:
                            if obs_label not in nl_label_dct:
                                noise_labels[obs_label] = obs_peak
                    else:
                        pass  # skip lower signals

        class_rank_obs_dct = rank_obs_dct.get("class", {})
        raw_class_obs_df_lst = []
        if class_rank_obs_dct:
            for c_r in class_rank_obs_dct:
                c_obs_df = class_rank_obs_dct[c_r].get("obs_df", pd.DataFrame)
                if not c_obs_df.empty:
                    c_obs_df["mz"] = class_rank_obs_dct[c_r].get("obs_mz", 0)
                    c_obs_df["i"] = class_rank_obs_dct[c_r].get("obs_i", 0)
                    raw_class_obs_df_lst.append(c_obs_df)

        class_obs_df_lst = [
            c_df for c_df in raw_class_obs_df_lst if isinstance(c_df, pd.DataFrame)
        ]

        if class_obs_df_lst:
            class_obs_df = pd.concat(class_obs_df_lst)
            class_obs_df.sort_values(by="i", ascending=False, inplace=True)
            class_obs_df.drop_duplicates(subset="label", keep="first", inplace=True)
            class_labels = list(class_label_dct.keys())
            matched_class_obs_df = class_obs_df[
                class_obs_df["label"].isin(class_labels)
            ]
            for c_i, c_r in matched_class_obs_df.iterrows():
                signal_labels[c_r["label"]] = [c_r["mz"], c_r["i"]]
                observed_specific_peaks[c_r["label"]] = {"ppm": c_r["ppm"]}
        else:
            pass

        unique_obs_names = list(set(observed_residues))

        has_all_residues = False
        if len(unique_obs_names) >= unique_res:
            has_all_residues = True
        # if pr_rank_score > 0:
        #     print(f"rank score: {pr_rank_score} | {pr_name} # {'; '.join(unique_obs_names)}")
        checked_noise_labels = {}
        for nl_lb in noise_labels:
            if nl_lb in signal_labels:
                pass
            else:
                checked_noise_labels[nl_lb] = noise_labels[nl_lb]

        if pr_rank_score > 100:
            pr_rank_score = 100
        if pr_rank_score >= rank_score_threshold:
            rank_score_results[pr_candidate] = {
                "name": pr_name,
                "label": pr_candidate,
                "neutral_formula": pr_candidate_info.get("neutral_formula", ""),
                "charged_formula": pr_candidate_info.get("formula", ""),
                "adduct": pr_candidate_info.get("adduct", ""),
                "mz": pr_candidate_info.get("mz", 0),
                "rank_score": pr_rank_score,
                "signal_labels": signal_labels,
                "noise_labels": checked_noise_labels,
                "observed_fragments": observed_fragments,
                "observed_neutral_losses": observed_neutral_losses,
                "observed_specific_peaks": observed_specific_peaks,
                "observed_residues": observed_residues,
                "ms2_df": ms2_df,
                "has_all_residues": has_all_residues,
            }
    else:
        pass

    return rank_score_results

def hunt_msp_score(
    pr_candidate: dict, ms2_df: pd.DataFrame, score_summary: dict, ms2_ppm: int = 50
):
    msp_score = 0
    msp_signal_labels = {}
    msp_absent_labels = {}
    pr_msp_dct = pr_candidate.get("msp", {})
    msp_obs_df_lst = []
    base_peak_i = ms2_df["i"].max()
    msp_factor_i = 0.001 * base_peak_i
    if pr_msp_dct:
        msp_df = pd.DataFrame(pr_msp_dct)
        msp_df.sort_values(by="mz", ascending=True, inplace=True)
        msp_chk_lst = list(zip(msp_df["mz"], msp_df["i"], msp_df["label"]))
        for chk_peak in msp_chk_lst:
            is_msp_matched = False
            mz = chk_peak[0]
            mz_l = mz * (1 - ms2_ppm * 0.000001)
            mz_h = mz * (1 + ms2_ppm * 0.000001)

            tmp_df = ms2_df.query(f"{mz_l} <= mz <= {mz_h}").copy()
            if tmp_df.shape[0] == 1:
                msp_obs_df_lst.append(tmp_df)
                is_msp_matched = True
            elif tmp_df.shape[0] > 1:
                tmp_df = tmp_df.sort_values(by="i", ascending=False)
                msp_obs_df_lst.append(tmp_df.head(1))
                is_msp_matched = True
            else:
                tmp_df = pd.DataFrame(data={"mz": [0.0], "i": [0.0]})
                msp_obs_df_lst.append(tmp_df)

            if is_msp_matched:
                msp_signal_labels[chk_peak[2]] = [
                    round(chk_peak[0], 4),
                    chk_peak[1] * msp_factor_i,
                ]
            else:
                msp_absent_labels[chk_peak[2]] = [
                    round(chk_peak[0], 4),
                    chk_peak[1] * msp_factor_i,
                ]

        if msp_obs_df_lst:
            obs_score_df = pd.concat(msp_obs_df_lst)

            # fit theo i to obs i
            obs_i_max = obs_score_df["i"].max()
            pr_msp_dct["abs_i"] = [i * 0.001 * obs_i_max for i in pr_msp_dct["i"]]

            # reduce matrix to 1D array
            obs_flat = get_msp_vectors(
                {
                    "mz": obs_score_df["mz"].values.tolist(),
                    "i": obs_score_df["i"].values.tolist(),
                }
            )
            lib_flat = get_msp_vectors(
                {
                    "mz": msp_df["mz"].values.tolist(),
                    "i": msp_df["i"].values.tolist(),
                }
            )
            # get cosine similarity
            if lib_flat and len(lib_flat) == len(obs_flat) and sum(lib_flat) > 0:
                msp_score = 100 * (
                    (1 - spatial.distance.cosine(obs_flat, lib_flat)) ** 2
                )
                msp_score = round(msp_score, 1)
        else:
            msp_score = 0

        if 0 <= msp_score <= 100:
            pass
        elif msp_score > 100:
            msp_score = 100
        else:
            msp_score = 0

    score_summary["msp_score"] = msp_score

    score_summary["signal_labels"] = msp_signal_labels
    score_summary["absent_labels"] = msp_absent_labels

    return score_summary

def hunt_fingerprint_score(
    pr_candidate: dict,
    ms2_df: pd.DataFrame,
    score_summary: dict,
    ms2_ppm: int = 50,
    fp_threshold: float = 0.01,
    amplify: float = 1.25,
):
    fingerprint_score = 0
    # signal_peaks = score_summary.get("signal_peaks", {})

    fp_signal_labels = {}
    fp_absent_labels = {}

    fp_lib_lst = []
    fp_obs_lst = []

    max_i = ms2_df["i"].max()
    fp_threshold = max_i * fp_threshold
    pr_fingerprint_lst = sorted(pr_candidate.get("fingerprints", []))

    for mz in pr_fingerprint_lst:
        is_fp_matched = False
        fp_lib_lst.append(1)
        mz_l = mz * (1 - ms2_ppm * 0.000001)
        mz_h = mz * (1 + ms2_ppm * 0.000001)

        tmp_df = ms2_df.query(f"{mz_l} <= mz <= {mz_h} and i >= {fp_threshold}").copy()

        if tmp_df.shape[0] == 1:
            fp_obs_lst.append(1)
            is_fp_matched = True
        elif tmp_df.shape[0] > 1:
            fp_obs_lst.append(1)
            is_fp_matched = True
        else:
            fp_obs_lst.append(0)
            is_fp_matched = False

        if is_fp_matched:
            fp_signal_labels[f"fp@{mz:.2f}"] = [
                round(mz, 4),
                1,
            ]
        else:
            fp_absent_labels[f"fp@{mz:.2f}"] = [
                round(mz, 4),
                0,
            ]

    sum_fp_lib_count = sum(fp_lib_lst)
    fp_count_amplify = 1 + 0.1 * (sum_fp_lib_count - 10)
    if sum_fp_lib_count > 0:
        fingerprint_score = (
            100
            * (1 - spatial.distance.cosine(np.array(fp_obs_lst), np.array(fp_lib_lst)))
            * amplify
            * fp_count_amplify
        )
        fingerprint_score = round(fingerprint_score, 1)
    if not fingerprint_score > 0:
        fingerprint_score = 0
    if fingerprint_score > 100:
        fingerprint_score = 100

    score_summary["fingerprint_score"] = fingerprint_score
    score_summary["signal_labels"] = fp_signal_labels
    score_summary["absent_labels"] = fp_absent_labels

    return score_summary

def hunt_snr_score(
    score_summary: dict,
    amplify_factor: float = 10.4795,
):
    snr_score = 0
    # raw_signal_peaks = score_summary.get("signal_labels", [{}])
    # raw_noise_peaks = score_summary.get("noise_labels", [{}])
    # signal_labels = dict(ChainMap(*raw_signal_peaks))
    # noise_labels = dict(ChainMap(*raw_noise_peaks))
    #
    # checked_noise_labels = {}
    # for nl_lb in noise_labels:
    #     if nl_lb in signal_labels:
    #         pass
    #     else:
    #         checked_noise_labels[nl_lb] = noise_labels[nl_lb]

    signal_df = pd.concat(
        [
            pd.DataFrame(s_dct).T
            for s_dct in score_summary.get("signal_labels", None)
            if s_dct is not None
        ]
    )
    pre_noise_df = pd.concat(
        [
            pd.DataFrame(n_dct).T
            for n_dct in score_summary.get("noise_labels", None)
            if n_dct is not None
        ]
    )
    signal_df.reset_index(inplace=True)
    signal_df.columns = ["label", "mz", "i"]
    signal_df.sort_values(by="i", ascending=False, inplace=True)
    signal_df.drop_duplicates(subset="label", keep="first", inplace=True)
    signal_df.drop_duplicates(subset="mz", keep="first", inplace=True)
    signal_df.sort_values(by="mz", ascending=False, inplace=True)
    signal_peaks = list(
        zip(signal_df["mz"].values.tolist(), signal_df["i"].values.tolist())
    )
    score_summary["signal_labels"] = dict(
        zip(signal_df["label"].values.tolist(), signal_peaks)
    )
    signal_sum_i = signal_df["i"].sum()

    if not pre_noise_df.empty:
        pre_noise_df.reset_index(inplace=True)
        pre_noise_df.columns = ["label", "mz", "i"]
        noise_df = pre_noise_df[~pre_noise_df["label"].isin(signal_df["label"])].copy()
        noise_df.sort_values(by="i", ascending=False, inplace=True)
        noise_df.drop_duplicates(subset="label", keep="first", inplace=True)
        noise_df.sort_values(by="mz", ascending=False, inplace=True)
        noise_peaks = list(
            zip(noise_df["mz"].values.tolist(), noise_df["i"].values.tolist())
        )
        score_summary["noise_labels"] = dict(
            zip(noise_df["label"].values.tolist(), noise_peaks)
        )
        noise_sum_i = noise_df["i"].sum()
    else:
        score_summary["noise_labels"] = {}
        noise_sum_i = 1

    if not noise_sum_i > 0:
        noise_sum_i = 1

    # use the SNR equation SNR = 20 * log10(signal/noise)
    # snr_score = 20 * math.log10((signal_sum_i / noise_sum_i))
    # set s/n == 25 --> SNR_SCORE = 100
    # default 3.5767 = 100 / (20 * math.log10(25)) --> 3.5767
    # if set s/n == 3 --> SNR_SCORE = 100
    # default 3.5767 = 100 / (20 * math.log10(3)) --> 10.4795
    sn_ratio = signal_sum_i / noise_sum_i
    snr_score = amplify_factor * 20 * math.log10(sn_ratio)

    if snr_score < 0:
        snr_score = 0.0
    elif snr_score > 100.0:
        snr_score = 100
    else:
        snr_score = round(snr_score, 1)

    score_summary["snr_score"] = snr_score

    return score_summary

def get_total_score(score_info: dict):
    score_lst = []
    score_name_lst = [
        "isotope_score",
        "rank_score",
        "msp_score",
        "fingerprint_score",
        "snr_score",
    ]
    if "msp_score" in score_info:
        msp_score = score_info.get("msp_score", 0)
        if msp_score <= 0:
            score_name_lst.remove("msp_score")

    for score_name in score_name_lst:
        tmp_score = score_info.get(score_name, 0)
        if tmp_score > 0:
            score_lst.append(tmp_score)

    total_score = round(sum(score_lst) / len(score_name_lst), 1)

    return total_score


from urllib.error import URLError

from django.conf import settings
from django.core.mail import send_mail

from apps import celery_app
from apps.proxy.models import ProxyNode, UserTrafficLog
from apps.sspanel import models as m
from apps.utils import get_current_datetime

@celery_app.task
def debug_task():
    u = m.User.objects.first()
    print("i am in debug", u)

@celery_app.task
def sync_user_traffic_task(node_id, data):
    """
    这个接口操作比较重，所以为了避免发信号
    所有写操作都需要用BULK的方式
    1 更新节点流量
    2 更新用户流量
    3 记录节点在线IP
    4 关闭超出流量的节点
    """
    node: ProxyNode = ProxyNode.get_or_none(node_id)
    if not node:
        return
    node_total_traffic = 0
    log_time = get_current_datetime()
    user_model_list = []
    trafficlog_model_list = []
    traffic_data = data.get("data", [])

    # load user in batch
    user_ids = []
    for user_data in traffic_data:
        user_id = user_data["user_id"]
        user_ids.append(user_data["user_id"])
    user_map = {}
    for u in m.User.objects.filter(id__in=user_ids):
        user_map[u.id] = u

    for user_data in traffic_data:
        user_id = int(user_data["user_id"])
        u = int(int(user_data["upload_traffic"]) * node.enlarge_scale)
        d = int(int(user_data["download_traffic"]) * node.enlarge_scale)
        # 个人流量增量
        user = user_map[user_id]
        user.download_traffic += d
        user.upload_traffic += u
        user.last_use_time = log_time
        user_model_list.append(user)
        # 个人流量记录
        trafficlog_model_list.append(
            UserTrafficLog(
                proxy_node=node,
                user=user,
                download_traffic=u,
                upload_traffic=d,
                ip_list=user_data.get("ip_list", []),
            )
        )
        # 节点流量增量
        node_total_traffic += u + d

    if not traffic_data:
        # NOTE add blank log to show node is online
        trafficlog_model_list.append(UserTrafficLog(proxy_node=node))
    # 节点流量记录
    node.used_traffic += node_total_traffic
    if node.overflow:
        node.enable = False
    node.current_used_download_bandwidth_bytes = int(data.get("download_bandwidth", 0))
    node.current_used_upload_bandwidth_bytes = int(data.get("upload_bandwidth", 0))
    node.save(
        update_fields=[
            "used_traffic",
            "enable",
            "current_used_download_bandwidth_bytes",
            "current_used_upload_bandwidth_bytes",
        ]
    )
    # 用户流量
    m.User.objects.bulk_update(
        user_model_list,
        [
            "download_traffic",
            "upload_traffic",
            "last_use_time",
        ],
    )
    # 流量记录
    UserTrafficLog.objects.bulk_create(trafficlog_model_list)

@celery_app.task
def check_user_state_task():
    """检测用户状态，将所有账号到期的用户状态重置"""
    m.User.check_and_disable_expired_users()
    m.User.check_and_disable_out_of_traffic_user()

@celery_app.task
def auto_reset_free_user_traffic_task():
    """重置所有免费用户流量"""
    for user in m.User.objects.filter(level=0):
        user.reset_traffic(settings.DEFAULT_TRAFFIC)

@celery_app.task
def reset_node_traffic_task():
    """重置节点使用流量"""
    for node in ProxyNode.objects.all():
        node.used_traffic = 0
        node.save()

@celery_app.task
def make_up_lost_order_task():
    """定时补单"""
    try:
        m.UserOrder.make_up_lost_orders()
    except URLError:
        # Note请求支付宝挂了，等下次retry就好
        pass

@celery_app.task
def clean_traffic_log_task():
    """清空七天前的所有流量记录"""
    dt = get_current_datetime().subtract(days=7)
    query = UserTrafficLog.objects.filter(created_at__lt=dt)
    count = query._raw_delete(query.db)
    print(f"UserTrafficLog  removed count:{count}")

@celery_app.task
def send_mail_to_users_task(user_id_list, subject, message):
    users = m.User.objects.filter(id__in=user_id_list)
    address = [user.email for user in users]
    if send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, address):
        logs = [
            m.EmailSendLog(user=user, subject=subject, message=message)
            for user in users
        ]
        m.EmailSendLog.objects.bulk_create(logs)
        print(f"send email success user: address: {address}")
    else:
        raise Exception(f"Could not send mail {address} subject: {subject}")

@celery_app.task
def close_stale_tickets_task():
    m.Ticket.close_stale_tickets()


import ctypes
import os
import sys
import shlex
import tarfile
import time
from datetime import datetime
from subprocess import Popen

import psutil

from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log
from .commands import CommandType

trial_output_path_name = ".nni"

class Trial:
    def __init__(self, args, data):
        self.process = None
        self.data = data
        self.args = args
        self.command_channel = args.command_channel
        self.trial_syslogger_stdout = None

        global NNI_TRIAL_JOB_ID
        self.id = data["trialId"]
        if self.id is None:
            raise Exception("trial_id is not found in %s" % data)
        os.environ['NNI_TRIAL_JOB_ID'] = self.id
        NNI_TRIAL_JOB_ID = self.id

        # for multiple nodes. If it's None, it means single node.
        self.node_id = args.node_id
        if self.node_id is None:
            self.name = self.id
        else:
            self.name = "%s_%s" % (self.id, self.node_id)

    def run(self):
        # redirect trial's stdout and stderr to syslog
        self.trial_syslogger_stdout = RemoteLogger(self.args.nnimanager_ip, self.args.nnimanager_port, 'trial', StdOutputType.Stdout,
                                                   self.args.log_collection, self.id, self.args.command_channel)

        nni_log(LogType.Info, "%s: start to run trial" % self.name)

        trial_working_dir = os.path.realpath(os.path.join(os.curdir, "..", "..", "trials", self.id))
        self.trial_output_dir = os.path.join(trial_working_dir, trial_output_path_name)
        trial_code_dir = os.path.join(trial_working_dir, "code")
        trial_nnioutput_dir = os.path.join(trial_working_dir, "nnioutput")

        environ = os.environ.copy()
        environ['NNI_TRIAL_SEQ_ID'] = str(self.data["sequenceId"])
        environ['NNI_OUTPUT_DIR'] = os.path.join(trial_working_dir, "nnioutput")
        environ['NNI_SYS_DIR'] = trial_working_dir
        self.working_dir = trial_working_dir

        # prepare code and parameters
        prepared_flag_file_name = os.path.join(trial_working_dir, "trial_prepared")
        if not os.path.exists(trial_working_dir):
            os.makedirs(trial_working_dir, exist_ok=True)

            os.makedirs(self.trial_output_dir, exist_ok=True)
            os.makedirs(trial_nnioutput_dir, exist_ok=True)
            # prepare code
            os.makedirs(trial_code_dir, exist_ok=True)
            with tarfile.open(os.path.join("..", "nni-code.tar.gz"), "r:gz") as tar:
                tar.extractall(trial_code_dir)

            # save parameters
            nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, self.data["parameter"]["value"]))
            parameter_file_name = os.path.join(trial_working_dir, "parameter.cfg")
            with open(parameter_file_name, "w") as parameter_file:
                parameter_file.write(self.data["parameter"]["value"])

            # ready flag
            with open(prepared_flag_file_name, "w") as prepared_flag_file:
                prepared_flag_file.write("%s" % (int(datetime.now().timestamp() * 1000)))

        # make sure code prepared by other node.
        if self.node_id is not None:
            while True:
                if os.path.exists(prepared_flag_file_name):
                    break
                time.sleep(0.1)

        trial_command = self.args.trial_command

        gpuIndices = self.data.get("gpuIndices")
        if (gpuIndices is not None):
            if sys.platform == "win32":
                trial_command = 'set CUDA_VISIBLE_DEVICES="%s " && call %s' % (gpuIndices, trial_command)
            else:
                trial_command = 'CUDA_VISIBLE_DEVICES="%s " %s' % (gpuIndices, trial_command)

        self.log_pipe_stdout = self.trial_syslogger_stdout.get_pipelog_reader()
        self.process = Popen(trial_command, shell=True, stdout=self.log_pipe_stdout,
                             stderr=self.log_pipe_stdout, cwd=trial_code_dir, env=dict(environ))
        nni_log(LogType.Info, '{0}: spawns a subprocess (pid {1}) to run command: {2}'.
                format(self.name, self.process.pid, shlex.split(trial_command)))

    def save_parameter_file(self, command_data):
        parameters = command_data["parameters"]
        file_index = int(parameters["index"])
        if file_index == 0:
            parameter_file_name = "parameter.cfg"
        else:
            parameter_file_name = "parameter_{}.cfg".format(file_index)
        parameter_file_name = os.path.join(self.working_dir, parameter_file_name)
        with open(parameter_file_name, "w") as parameter_file:
            nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, parameters["value"]))
            parameter_file.write(parameters["value"])

    def is_running(self):
        if (self.process is None):
            return False

        retCode = self.process.poll()
        # child worker process exits and all stdout data is read
        if retCode is not None and self.log_pipe_stdout.set_process_exit() and self.log_pipe_stdout.is_read_completed == True:
            # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError.
            # So covert it to int32.
            retCode = ctypes.c_long(retCode).value
            nni_log(LogType.Info, '{0}: subprocess terminated. Exit code is {1}.'.format(self.name, retCode))

            end_time = int(datetime.now().timestamp() * 1000)
            end_message = {
                "code": retCode,
                "time": end_time,
                "trial": self.id,
            }
            self.command_channel.send(CommandType.TrialEnd, end_message)
            self.cleanup()
            return False
        else:
            return True

    def kill(self, trial_id=None):
        if trial_id == self.id or trial_id is None:
            if self.process is not None:
                try:
                    nni_log(LogType.Info, "%s: killing trial" % self.name)
                    for child in psutil.Process(self.process.pid).children(True):
                        child.kill()
                    self.process.kill()
                except psutil.NoSuchProcess:
                    nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid))
                except Exception as ex:
                    nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex)))
            self.cleanup()

    def cleanup(self):
        nni_log(LogType.Info, "%s: clean up trial" % self.name)
        self.process = None
        if self.log_pipe_stdout is not None:
            self.log_pipe_stdout.set_process_exit()
            self.log_pipe_stdout = None
        if self.trial_syslogger_stdout is not None:
            self.trial_syslogger_stdout.close()
            self.trial_syslogger_stdout = None


a=input().split()
res=[]
for i in a:
	for j in range(0,len(i)):
		if i[j].isdigit():
			continue
		else:
			t=i[j].lower()
			if t not in res:
				b=ord(t)
				if (b>96 and b<123):
					res.append(t)
g=list(set(res))
if len(g)==26:
	print("yes")
else:
	print("no")