from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import threading
import time

import numpy as np

from tensorflow.contrib import lookup
from tensorflow.contrib.eager.python import datasets
from tensorflow.python.data.experimental.ops import threadpool
from tensorflow.python.data.experimental.ops import unique
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.eager import test
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import script_ops
from tensorflow.python.training import checkpoint_management
from tensorflow.python.training.tracking import util as trackable_utils

class IteratorTest(test.TestCase):

  def testBasic(self):
    got = []
    for t in datasets.Iterator(dataset_ops.Dataset.range(4)):
      got.append(t.numpy())
    self.assertAllEqual([0, 1, 2, 3], got)

  def testBasicOneShotIterator(self):
    got = []
    for t in dataset_ops.Dataset.range(4).make_one_shot_iterator():
      got.append(t.numpy())
    self.assertAllEqual([0, 1, 2, 3], got)

  def testBasicImplicitIterator(self):
    got = []
    for t in dataset_ops.Dataset.range(4):
      got.append(t.numpy())
    self.assertAllEqual([0, 1, 2, 3], got)

  def testGetNext(self):
    iterator = datasets.Iterator(dataset_ops.Dataset.range(4))
    self.assertEqual(0, iterator.get_next().numpy())
    self.assertEqual(1, iterator.get_next().numpy())
    self.assertEqual(2, iterator.get_next().numpy())
    self.assertEqual(3, iterator.get_next().numpy())
    with self.assertRaises(errors.OutOfRangeError):
      iterator.get_next()

  def testGetNextOneShotIterator(self):
    iterator = dataset_ops.Dataset.range(4).make_one_shot_iterator()
    self.assertEqual(0, iterator.get_next().numpy())
    self.assertEqual(1, iterator.get_next().numpy())
    self.assertEqual(2, iterator.get_next().numpy())
    self.assertEqual(3, iterator.get_next().numpy())
    with self.assertRaises(errors.OutOfRangeError):
      iterator.get_next()

  def testMultipleIteratorsOnTheSameDataset(self):
    ds = dataset_ops.Dataset.range(4)
    it1 = datasets.Iterator(ds)
    it2 = datasets.Iterator(ds)
    got = [x.numpy() for x in it1]
    self.assertAllEqual([0, 1, 2, 3], got)

    got = [x.numpy() for x in it2]
    self.assertAllEqual([0, 1, 2, 3], got)

  def testNestedOutputs(self):
    ds = dataset_ops.Dataset.zip(
        (dataset_ops.Dataset.range(4),
         dataset_ops.Dataset.zip(
             (dataset_ops.Dataset.range(4), dataset_ops.Dataset.range(4)))))
    total = 0
    # The Iterator will return a nested structure of Tensor objects.
    # Some funkiness to compare against simple integers.
    for (i, x) in enumerate(datasets.Iterator(ds)):
      want = (i, (i, i))
      got = (x[0].numpy(), (x[1][0].numpy(), x[1][1].numpy()))
      self.assertEqual(got, want)
      total += 1
    self.assertEqual(4, total)

  def testMapAndFilter(self):

    def even(x):
      return math_ops.equal(math_ops.mod(x, 2), 0)

    it = datasets.Iterator(
        dataset_ops.Dataset.range(8).map(math_ops.square).filter(even))
    got = [x.numpy() for x in it]
    self.assertAllEqual([0, 4, 16, 36], got)

  def testMapCaptureLookupTable(self):
    default_val = -1
    keys = constant_op.constant(['brain', 'salad', 'surgery'])
    values = constant_op.constant([0, 1, 2], dtypes.int64)
    table = lookup.HashTable(
        lookup.KeyValueTensorInitializer(keys, values), default_val)
    dataset = dataset_ops.Dataset.from_tensor_slices(
        ['brain', 'salad', 'surgery'])
    dataset = dataset.map(table.lookup)
    it = datasets.Iterator(dataset)
    got = [x.numpy() for x in it]
    self.assertAllEqual([0, 1, 2], got)

  def testMultipleIteratorsOnADatasetThatUsesFunctions(self):
    ds = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5,
                                                 6]).map(math_ops.square)

    got1 = [x.numpy() for x in datasets.Iterator(ds)]
    self.assertAllEqual([1, 4, 9, 16, 25, 36], got1)
    got2 = [x.numpy() for x in datasets.Iterator(ds)]
    self.assertAllEqual(got1, got2)

  def assertSparseValuesEqual(self, a, b):
    self.assertAllEqual(a.indices, b.indices)
    self.assertAllEqual(a.values, b.values)
    self.assertAllEqual(a.dense_shape, b.dense_shape)

  def testSparseTensorElements(self):
    components = (sparse_tensor.SparseTensorValue(
        indices=np.array([[0, 0], [1, 0], [2, 0]]),
        values=np.array([0, 0, 0]),
        dense_shape=np.array([3, 1])),
                  sparse_tensor.SparseTensorValue(
                      indices=np.array([[0, 0], [1, 1], [2, 2]]),
                      values=np.array([1, 2, 3]),
                      dense_shape=np.array([3, 3])))

    expected = [
        (sparse_tensor.SparseTensorValue(
            indices=np.array([[0]]),
            values=np.array([0]),
            dense_shape=np.array([1])),
         sparse_tensor.SparseTensorValue(
             indices=np.array([[0]]),
             values=np.array([1]),
             dense_shape=np.array([3]))),
        (sparse_tensor.SparseTensorValue(
            indices=np.array([[0]]),
            values=np.array([0]),
            dense_shape=np.array([1])),
         sparse_tensor.SparseTensorValue(
             indices=np.array([[1]]),
             values=np.array([2]),
             dense_shape=np.array([3]))),
        (sparse_tensor.SparseTensorValue(
            indices=np.array([[0]]),
            values=np.array([0]),
            dense_shape=np.array([1])),
         sparse_tensor.SparseTensorValue(
             indices=np.array([[2]]),
             values=np.array([3]),
             dense_shape=np.array([3]))),
    ]

    for i, result in enumerate(
        datasets.Iterator(dataset_ops.Dataset.from_tensor_slices(components))):
      self.assertSparseValuesEqual(expected[i][0], result[0])
      self.assertSparseValuesEqual(expected[i][1], result[1])

  def testPyFunc(self):

    def my_map(inp):
      return [[x + 1 for x in inp]]

    ds = dataset_ops.Dataset.range(4).map(
        lambda x: script_ops.py_func(my_map, [[x]], dtypes.int64))
    got = [x.numpy() for x in datasets.Iterator(ds)]
    self.assertAllEqual([[1], [2], [3], [4]], got)

  def testTensorsPlacedOnDevice(self):
    ds = dataset_ops.Dataset.from_tensors([0., 1.])
    with ops.device(test.gpu_device_name()):
      x = datasets.Iterator(ds).next()
      x = math_ops.add(x, x)
    self.assertAllEqual([0., 2.], x.numpy())

  def testGpuTensor(self):
    ds = dataset_ops.Dataset.from_tensors([0., 1.])
    with ops.device(test.gpu_device_name()):
      for x in ds:
        y = math_ops.add(x, x)
    self.assertAllEqual([0., 2.], y.numpy())

  def testOverrideThreadPool(self):

    def get_thread_id(_):
      # Python creates a dummy thread object to represent the current
      # thread when called from an "alien" thread (such as a
      # `PrivateThreadPool` thread in this case). It does not include
      # the TensorFlow-given display name, but it has a unique
      # identifier that maps one-to-one with the underlying OS thread.
      return np.array(threading.current_thread().ident).astype(np.int64)

    for num_threads in [1, 2, 4, 8, 16]:

      dataset = (
          dataset_ops.Dataset.range(1000).map(
              lambda x: script_ops.py_func(get_thread_id, [x], dtypes.int64),
              num_parallel_calls=32).apply(unique.unique()))

      dataset = threadpool.override_threadpool(
          dataset,
          threadpool.PrivateThreadPool(
              num_threads, display_name='private_thread_pool_%d' % num_threads))

      thread_ids = []
      for next_element in datasets.Iterator(dataset):
        thread_ids.append(next_element)
      self.assertEqual(len(thread_ids), len(set(thread_ids)))
      self.assertGreater(len(thread_ids), 0)
      # NOTE(mrry): We don't control the thread pool scheduling, and
      # so cannot guarantee that all of the threads in the pool will
      # perform work.
      self.assertLessEqual(len(thread_ids), num_threads)

  def testSaveRestore(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
    dataset = dataset_ops.Dataset.from_tensor_slices(
        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
    dataset = dataset.map(math_ops.square).batch(2)
    # TODO(b/138399725): Re-enable default optimizations.
    options = dataset_ops.Options()
    options.experimental_optimization.apply_default_optimizations = False
    dataset = dataset.with_options(options)
    iterator = datasets.Iterator(dataset)
    checkpoint = trackable_utils.Checkpoint(iterator=iterator)
    self.assertAllEqual([1, 4], iterator.get_next().numpy())
    save_path = checkpoint.save(checkpoint_prefix)
    self.assertAllEqual([9, 16], iterator.get_next().numpy())
    self.assertAllEqual([25, 36], iterator.get_next().numpy())
    checkpoint.restore(save_path)
    self.assertAllEqual([9, 16], iterator.get_next().numpy())
    self.assertAllEqual([25, 36], iterator.get_next().numpy())

  def testSaveRestoreMultipleIterator(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
    dataset = dataset_ops.Dataset.from_tensor_slices(
        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
    dataset = dataset.map(math_ops.square).batch(2)
    # TODO(b/138399725): Re-enable default optimizations.
    options = dataset_ops.Options()
    options.experimental_optimization.apply_default_optimizations = False
    dataset = dataset.with_options(options)
    iterator_1 = datasets.Iterator(dataset)
    iterator_2 = datasets.Iterator(dataset)
    dataset_2 = dataset_ops.Dataset.range(10)
    iterator_3 = datasets.Iterator(dataset_2)

    checkpoint = trackable_utils.Checkpoint(
        iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3)
    self.assertAllEqual([1, 4], iterator_1.get_next().numpy())
    self.assertEqual(0, iterator_3.get_next().numpy())
    self.assertEqual(1, iterator_3.get_next().numpy())
    self.assertEqual(2, iterator_3.get_next().numpy())

    save_path = checkpoint.save(checkpoint_prefix)
    self.assertAllEqual([1, 4], iterator_2.get_next().numpy())
    self.assertAllEqual([9, 16], iterator_2.get_next().numpy())
    self.assertEqual(3, iterator_3.get_next().numpy())
    checkpoint.restore(save_path)
    self.assertAllEqual([9, 16], iterator_1.get_next().numpy())
    self.assertAllEqual([1, 4], iterator_2.get_next().numpy())
    self.assertEqual(3, iterator_3.get_next().numpy())

  def testRestoreExhaustedIterator(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
    dataset = dataset_ops.Dataset.range(3)
    iterator = datasets.Iterator(dataset)

    checkpoint = trackable_utils.Checkpoint(iterator=iterator)
    self.assertEqual(0, iterator.get_next().numpy())
    self.assertEqual(1, iterator.get_next().numpy())
    save_path = checkpoint.save(checkpoint_prefix)
    self.assertEqual(2, iterator.get_next().numpy())
    checkpoint.restore(save_path)
    self.assertEqual(2, iterator.get_next().numpy())

  def testRestoreInReconstructedIterator(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
    dataset = dataset_ops.Dataset.range(10)
    for i in range(5):
      iterator = datasets.Iterator(dataset)
      checkpoint = trackable_utils.Checkpoint(iterator=iterator)
      checkpoint.restore(
          checkpoint_management.latest_checkpoint(checkpoint_directory))
      for j in range(2):
        self.assertEqual(i * 2 + j, iterator.get_next().numpy())
      checkpoint.save(file_prefix=checkpoint_prefix)

class DatasetConstructorBenchmark(test.Benchmark):

  def benchmarkSliceRepeatBatchEager(self):
    input_size = 10000
    batch_size = 100
    num_epochs = 100

    input_data = np.random.randn(input_size)

    dataset = (
        dataset_ops.Dataset.from_tensor_slices(input_data).repeat(
            num_epochs).batch(batch_size))
    iterator = datasets.Iterator(dataset)

    ends = [time.time()]
    for _ in iterator:
      ends.append(time.time())

    deltas = np.ediff1d(ends)
    median_wall_time = np.median(deltas)
    print('Slice/repeat/batch eager input size: %d batch size: %d Median wall '
          'time per element: %f' % (input_size, batch_size, median_wall_time))
    self.report_benchmark(
        iters=len(deltas),
        wall_time=median_wall_time,
        name='benchmark_slice_repeat_batch_eager_input_%d_batch_%d' %
        (input_size, batch_size))

  def benchmarkSliceBatchCacheRepeatCallable(self):
    input_size = 10000
    batch_size = 100
    num_epochs = 100

    input_data = np.random.randn(input_size)

    dataset = (
        dataset_ops.Dataset.from_tensor_slices(input_data).batch(
            batch_size).cache().repeat(num_epochs))
    iterator = datasets.Iterator(dataset)

    ends = [time.time()]
    for _ in iterator:
      ends.append(time.time())

    deltas = np.ediff1d(ends)
    median_wall_time = np.median(deltas)
    print('Slice/batch/cache/repeat eager input size: %d batch size: %d Median '
          'wall time per element: %f' %
          (input_size, batch_size, median_wall_time))
    self.report_benchmark(
        iters=len(deltas),
        wall_time=median_wall_time,
        name='benchmark_slice_batch_cache_repeat_eager_input_%d_batch_%d' %
        (input_size, batch_size))

if __name__ == '__main__':
  test.main()


#!/usr/bin/env python
import math
import sys
import numpy as np
from scipy.ndimage import gaussian_filter1d

# this function is a copy of Iterative-1d-gm.py
# BUT: get as parameter initial dx
def iterative1d(x1, x2, dx):
    # match signals length
    if len(x1) > len(x2):
        x1 = x1[:len(x2)]
    else:
        x2 = x2[:len(x1)]

    # update iteratively
    threshold = 0.0001
    dx_update = threshold
    while np.abs(dx_update) >= 0.0001:
        n = len(x1) - math.ceil(np.abs(dx))  # overlap area size

        if dx < 0:
            # shift x2 right
            dx_abs = np.abs(dx)
            dx_frac = dx_abs - math.floor(dx_abs)
            x2s = x2[math.floor(dx_abs):]  # integer shift
            x2s = np.array([(x2s[i] + (x2s[i + 1] - x2s[i]) * dx_frac) for i in range(n - 1)])  # fraction shift
            x1c = x1[0:-math.floor(dx_abs) - 1]  # cut unnecessary part of x1 (keep just the overlap area)
        else:  # dx >= 0
            # shift x2 left
            dx_frac = dx - math.floor(dx)
            x2s = x2[:-math.floor(dx)] if math.floor(dx) > 0 else x2  # integer shift
            x2s = np.array([(x2s[i] - (x2s[i] - x2s[i - 1]) * dx_frac) for i in range(1, n)])  # fraction shift
            x1c = x1[math.floor(dx) + 1:]  # cut unnecessary part of x1 (keep just the overlap area)

        # build and solve linear equation
        a = np.array([(x1c[i + 1] - x1c[i - 1]) for i in range(1, n - 2)]).reshape((n - 3, 1))  # derivative
        b = np.array([(x2s[i] - x1c[i]) for i in range(1, n - 2)]).reshape((n - 3, 1))  # difference
        dx_update = np.linalg.solve(a.T @ a, a.T @ b)[0][0]  # least square
        dx += dx_update

    return dx

def main(argv):
    # get signals to numpy array
    # NOTE: assuming names are 'x1', 'x2'
    x1 = gaussian_filter1d(np.load(argv[0])['x1'], sigma=1)
    x2 = gaussian_filter1d(np.load(argv[1])['x2'], sigma=1)

    # decimate to 1/4 resolution and get initial dx4
    dx = iterative1d(gaussian_filter1d(x1, sigma=1)[::4], gaussian_filter1d(x2, sigma=1)[::4], 0)
    # decimate to 1/2 resolution, start from 2*dx4 and get dx2
    dx = iterative1d(gaussian_filter1d(x1, sigma=1)[::2], gaussian_filter1d(x2, sigma=1)[::2], dx * 2)
    # original resolution, start from 2*dx2 and get dx
    dx = iterative1d(x1, x2, dx * 2)

    print('dx = %.2f' % dx)

if __name__ == '__main__':
    main(sys.argv[1:])


import paho.mqtt.client as mqtt

def on_connect(client, userdata, flags, rc):
	print("Connected with result code: " + str(rc))
	client.publish("my/test", "Hey mate")

client = mqtt.Client()
client.on_connect = on_connect
client.username_pw_set("michael", "P@ssw0rd")
client.connect("3.88.162.190", 1883, 60)

client.loop_forever()


import re
from subprocess import Popen, PIPE
from board.Board import BLACK, NONE, getOtherColor, getPieceSymbol, WHITE, getDirection, Board
from move.Move import MoveNode
from move.MovementFactory import generate_moves
from players.Player import Player

class GnuPlayer(Player):
    def __init__(self, color):
        self.name = "gnubg"
        if color != WHITE:
            raise Exception("Gnubg should be white to avoid confusion")
        self.color = color

    def get_move(self, backgammon):
        moves = generate_moves(backgammon.board, self.color, backgammon.dice)
        if len(moves) == 1:
            return moves.pop()
        export_to_snowietxt(backgammon,
                            r"C:\users\tatem\onedrive\documents\my stuff\senior\ai\course-project-tate\interfacing\to_gnu.txt")
        p = Popen([r"C:\Program Files (x86)\gnubg\gnubg-cli.exe", "-q", "-t", "-c",
                   r"C:\users\tatem\onedrive\documents\my stuff\senior\ai\course-project-tate\interfacing\command.txt"],
                  stdout=PIPE, stdin=PIPE)
        outs, errs = p.communicate(timeout=10)
        p.wait()
        f = re.search(r"gnubg moves (.*)" + re.escape("."), outs.decode('ascii'))
        if not f:
            name = "couldn't find text of move, output below"
            if "gnubg offers to resign" in outs.decode('ascii'):
                board = backgammon.board
        else:
            name = f.group(1) + " from gnubg's point of view"
            board = import_from_snowietxt(
                r"C:\users\tatem\onedrive\documents\my stuff\senior\ai\course-project-tate\interfacing\from_gnu.txt")
        move = MoveNode(name, board, deep=0)
        return move

    def __str__(self):
        return self.name + " (" + getPieceSymbol(self.color) + ")"

def export_to_snowietxt(backgammon, outfile):
    current_player = backgammon.players[backgammon.on_roll]
    if type(current_player) != GnuPlayer:
        return False
    current_color = current_player.color
    direction = getDirection(current_color)
    other_player = backgammon.players[(backgammon.on_roll + 1) % 2]

    string = ""

    match_length = 0
    string += str(match_length) + ";"

    jacoby = 0
    string += str(jacoby) + ";0;1;"

    current_player = 0
    string += str(current_player) + ";"

    name = other_player.name
    string += "gnubg;" + str(name) + ";"

    crawford = 0
    string += str(crawford) + ";"

    score_0, score_1 = 0, 0
    string += str(score_0) + ";" + str(score_1) + ";"

    cube_value = backgammon.board.doubleCube
    string += str(cube_value) + ";"

    cube_possesion = -1  # don't give the computer the doubling cube
    string += str(cube_possesion) + ";"

    other_on_bar = backgammon.board.numBar(getOtherColor(current_color))
    string += str(other_on_bar) + ";"

    points = reversed(backgammon.board.pointsContent[1:-1])
    points = [-direction * point for point in points]
    for point in points:
        string += str(point) + ";"

    current_on_bar = backgammon.board.numBar(current_color)
    string += str(current_on_bar) + ";"

    die_1, die_2 = backgammon.dice.die1, backgammon.dice.die2
    string += str(die_1) + ";" + str(die_2) + ";"

    while True:
        try:
            with open(outfile, "w") as f:
                f.write(string)
                break
        except PermissionError:
            print("encountered a permission error. attempting to write again")
            continue

def import_from_snowietxt(infile):
    # TODO: encountering permission denied which prevents move reading
    while True:
        try:
            with open(infile) as f:
                string = f.readline()
                break
        except PermissionError:
            print("encountered a permission error. attempting to read again")
            continue

    vals = string.split(";")
    if vals[4] != "1" or vals[6] != "gnubg":
        raise Exception("Invalid file")

    b = Board()
    b.doubleCube = int(vals[10])
    b.doublePossession = NONE if vals[11] == "0" else BLACK if vals[11] == "1" else WHITE
    b.blackCheckersTaken = int(vals[37])
    b.whiteCheckersTaken = int(vals[12])
    b.blackCheckers = set()
    b.whiteCheckers = set()
    black_count = 15 - b.blackCheckersTaken
    white_count = -15 + b.whiteCheckersTaken
    for point, v in enumerate(vals[13:37], 1):
        v = int(v)
        b.pointsContent[point] = v
        if v < 0:
            b.whiteCheckers.add(point)
            white_count -= v
        elif v > 0:
            b.blackCheckers.add(point)
            black_count -= v
    b.pointsContent[0] = black_count
    b.pointsContent[25] = white_count
    return b


# -*- coding: utf-8 -*-
"""
Created on 2018/1/10 16:34

@author: JERRY
"""
from threading import Thread
import ctypes
import inspect
from time import sleep

class myThread(Thread):

    def __init__(self, name = None, target=None, *args, **kwargs):
        Thread.__init__(self, name=name, target=target, *args, **kwargs)

    def _async_raise(self, exctype):
        """raises the exception, performs cleanup if needed"""
        tid = ctypes.c_long(self.ident)
        if not inspect.isclass(exctype):
            exctype = type(exctype)
        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
        if res == 0:
            raise ValueError("invalid thread id")
        elif res != 1:
            # """if it returns a number greater than one, you're in trouble,
            # and you should call it again with exc=NULL to revert the effect"""
            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
            raise SystemError("PyThreadState_SetAsyncExc failed")

    def close(self):
        self._async_raise(SystemExit)
        print(self.name + ' has been killed!')

if __name__ == '__main__':
    def t1(x = 0, y = 0, z = 0):
        while True:
            print(x+y+z)
            sleep(1)

    param = {'x':1,'z':2}

    test = myThread(name='test',target=t1,kwargs=param)
    test.start()


#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Py2-backwards compatibility
try:
  basestring
except NameError:
  basestring = str

import copy
import numpy as np

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class NamedAxisError(Exception):
    pass

class KeyStruct(object):
    """A slightly enhanced version of a struct-like class with named key access.

    Examples
    --------

    >>> a = KeyStruct()
    >>> a.x = 1
    >>> a['x']
    1
    >>> a['y'] = 2
    >>> a.y
    2
    >>> a[3] = 3
    Traceback (most recent call last):
      ...
    TypeError: hasattr(): attribute name must be string

    >>> b = KeyStruct(x=1, y=2)
    >>> b.x
    1
    >>> b['y']
    2
    >>> b['y'] = 4
    Traceback (most recent call last):
      ...
    AttributeError: KeyStruct already has atribute 'y'

    """
    def __init__(self, **kw):
        self.__dict__.update(kw)

    def __getitem__(self, key):
        return self.__dict__[key]

    def __setitem__(self, key, val):
        if hasattr(self, key):
            raise AttributeError('KeyStruct already has atribute %s'%repr(key))
        self.__dict__[key] = val

    def __setattr__(self, key, val):
        self[key] = val

class AxesManager(object):
    """
    Class to manage the logic of the datarray.axes object.

    >>> A = DataArray(np.random.randn(200, 4, 10), \
                axes=('date', ('stocks', ('aapl', 'ibm', 'goog', 'msft')), 'metric'))
    >>> isinstance(A.axes, AxesManager)
    True

    At a basic level, AxesManager acts like a sequence of axes:

    >>> A.axes # doctest:+ELLIPSIS
    (Axis(name='date', index=0, labels=None), ..., Axis(name='metric', index=2, labels=None))
    >>> A.axes[0]
    Axis(name='date', index=0, labels=None)
    >>> len(A.axes)
    3
    >>> A.axes[4]
    Traceback (most recent call last):
        ...
    IndexError: Requested axis 4 out of bounds

    Each axis is accessible as a named attribute:

    >>> A.axes.stocks
    Axis(name='stocks', index=1, labels=('aapl', 'ibm', 'goog', 'msft'))

    An axis can be indexed by integers or ticks:

    >>> np.all(A.axes.stocks['aapl':'goog'] == A.axes.stocks[0:2])
    True

    >>> np.all(A.axes.stocks[0:2] == A[:,0:2,:])
    True

    Axes can also be accessed numerically:

    >>> A.axes[1] is A.axes.stocks
    True

    Calling the AxesManager with string arguments will return an
    :py:class:`AxisIndexer` object which can be used to restrict slices to
    specified axes:

    >>> Ai = A.axes('stocks', 'date')
    >>> np.all(Ai['aapl':'goog', 100] == A[100, 0:2])
    True

    You can also mix axis names and integers when calling AxesManager.
    (Not yet supported.)

    # >>> np.all(A.axes(1, 'date')['aapl':'goog',100:200] == A[100:200, 0:2])
    # True
    """

    # The methods of this class use object.__getattribute__ to avoid a
    # potential collision between axis names and the internal instance
    # variables
    def __init__(self, arr, axes):
        self._arr = arr
        self._axes = tuple(axes)
        self._namemap = dict((ax.name,i) for i,ax in enumerate(axes))

    # This implements darray.axes.an_axis_name
    def __getattribute__(self, name):
        namemap = object.__getattribute__(self, '_namemap')
        axes = object.__getattribute__(self, '_axes')
        try:
            return axes[namemap[name]]
        except KeyError:
            return object.__getattribute__(self, name)

    def __len__(self):
        return len(object.__getattribute__(self, '_axes'))

    def __repr__(self):
        return str(tuple(self))

    def __getitem__(self, n):
        """Return the axis object at integer index `n`

        Parameters
        ----------
        n : int
            Index of axis to be returned.

        Returns
        -------
        ax : :class:`Axis` instance
            The requested :py:class:`Axis`.

        Examples
        --------
        >>> A = DataArray([[1,2],[3,4]], 'ab'); A.axes[0] is A.axes.a
        True
        >>> A.axes[1] is A.axes.b
        True
        """
        if not isinstance(n, int):
            raise TypeError("AxesManager expects integer index")
        try:
            return object.__getattribute__(self, '_axes')[n]
        except IndexError:
            raise IndexError("Requested axis %i out of bounds" % n)

    def __eq__(self, other):
        """Test for equality between two axes managers. Two axes managers are
        equal if the axes they manage are equal and have the same order.

        Examples
        --------
        >>> A = DataArray([[1,2],[3,4]], 'ab')
        >>> B = DataArray([[7,8],[9,10]], 'ab')
        >>> C = DataArray([[7,8],[9,10]], 'cd')
        >>> D = DataArray([[1,2,3,4],[5,6,7,8]], 'ab')
        >>> A.axes == B.axes
        True
        >>> A.axes == C.axes
        False
        >>> A.axes == D.axes
        True

        Parameters
        ----------
        other : any

        Returns
        -------
        out : bool

        """
        if not isinstance(other, AxesManager):
            return False
        axes = object.__getattribute__(self, '_axes')
        return axes == other._axes

    def __call__(self, *args):
        """Return an axis indexer object based on the supplied arguments.

        Parameters
        ----------
        args : sequence of strs
            A sequence of axis names.

        Returns
        -------
        If len(args)==1, the axis itself is returned. Otherwise, an
        :py:class:`AxisIndexer` which indexes over specified axes.

        """
        namemap = object.__getattribute__(self, '_namemap')
        axes = object.__getattribute__(self, '_axes')
        arr = object.__getattribute__(self, '_arr')
        if len(args) == 1:
            return axes[namemap[args[0]]]
        else:
            return AxisIndexer(arr, *args)

class AxisIndexer(object):
    """
    An object which holds a reference to a DataArray and a list of axes and
    allows slicing by those axes.
    """
    # XXX don't support mapped indexing yet...
    def __init__(self, arr, *args):
        self.arr = arr
        self.axes = args
        axis_set = set(args)
        self._axis_map = [self.axes.index(axis.name) if axis.name in self.axes else None
            for axis in arr.axes]

    def __getitem__(self, item):
        if not isinstance(item, tuple):
            item = item,

        if len(item) != len(self.axes):
            raise ValueError("Incorrect slice length")

        slicer = tuple(
            item[self._axis_map[i]]
                if self._axis_map[i] is not None
                else slice(None, None, None)
            for i in range(len(self.arr.axes)))

        return self.arr[slicer]

class Axis(object):
    "Object to access a given axis of an array."
    # Key point: every axis contains a reference to its parent array!

    def __init__(self, name, index, parent_arr, labels=None):
        # Axis name should be a string or None
        if not isinstance(name, basestring) and name is not None:
            raise ValueError('name must be a string or None')
        self.name = name
        self.index = index
        self.parent_arr = parent_arr

        # If labels is not None, name should be defined
        if labels is not None and name is None:
            raise ValueError('labels only supported when Axis has a name')

        # This will raise if the labels are invalid:
        self._label_dict = self._validate_labels(labels)
        self.labels = labels

    def _copy(self, **kwargs):
        """
        Create a quick copy of this Axis without bothering to do
        label validation (these labels are already known as valid).

        Keyword args are replacements for constructor arguments

        Examples
        --------

        >>> a1 = Axis('time', 0, None, labels=[str(i) for i in range(10)])
        >>> a1
        Axis(name='time', index=0, labels=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
        >>> a2 = a1._copy(labels=a1.labels[3:6])
        >>> a2
        Axis(name='time', index=0, labels=['3', '4', '5'])
        >>> a1 == a2
        False
        """
        name = kwargs.pop('name', self.name)
        index = kwargs.pop('index', self.index)
        parent_arr = kwargs.pop('parent_arr', self.parent_arr)
        cls = self.__class__
        ax = cls(name, index, parent_arr)

        labels = kwargs.pop('labels', copy.copy(self.labels))
        ax.labels = labels
        if labels is not None and len(labels) != len(self.labels):
            ax._label_dict = dict( zip(labels, range( len(labels) )) )
        else:
            ax._label_dict = copy.copy(self._label_dict)
        return ax

    # A guaranteed-to-be-a-string version of the axis name, which lets us
    # disambiguate when multiple unnamed axes exist in an array (since they all
    # have None for name).
    @property
    def _sname(self):
        if self.name is not None:
            return str(self.name)
        else:
            return "_%d" %  self.index

    def _validate_labels(self, labels):
        """Validate constraints on labels.

        Ensure:

        - uniqueness
        - length
        - no label is an integer
        """
        if labels is None:
            return None

        nlabels = len(labels)
        # XXX maybe Axis labels should be validated in __array_finalize__?

        # Sanity check: the first dimension must match that of the parent array
        if self.parent_arr is not None \
               and nlabels != self.parent_arr.shape[self.index]:
            e = 'Dimension mismatch between labels and data at index %i' % \
                self.index
            raise ValueError(e)

        # Validate types -- using generator for short circuiting
        if any( (isinstance(t, int) for t in labels) ):
            raise ValueError('Labels cannot be integers')

        # Validate uniqueness
        t_dict = dict(zip(labels, range(nlabels)))
        if len(t_dict) != nlabels:
            raise ValueError('non-unique label values not supported')
        return t_dict

    def set_name(self, name):
        # XXX: This makes some potentially scary changes to the parent
        #      array. It may end up being an insidious bug.

        # Axis name should be a string or None
        if not isinstance(name, basestring) and name is not None:
            raise ValueError('name must be a string or None')
        self.name = name
        pa = self.parent_arr
        nd = pa.ndim
        newaxes = [pa.axes[i] for i in range(self.index)]
        newaxes += [self]
        newaxes += [pa.axes[i] for i in range(self.index+1,nd)]
        _set_axes(pa, newaxes)

    def __len__(self):
        return self.parent_arr.shape[self.index]

    def __eq__(self, other):
        """
        Axes are equal iff they have matching names and indices. They
        do not need to have matching labels.

        Parameters
        ----------
        other : ``Axis`` object
           Object to compare

        Returns
        -------
        tf : bool
           True if self == other

        Examples
        --------
        >>> ax = Axis('x', 0, np.arange(10))
        >>> ax == Axis('x', 0, np.arange(5))
        True
        >>> ax == Axis('x', 1, np.arange(10))
        False
        """
        if not isinstance(other, self.__class__):
            return False

        return self.name == other.name and self.index == other.index and \
               self.labels == other.labels

    def __repr__(self):
        return 'Axis(name=%r, index=%i, labels=%r)' % \
               (self.name, self.index, self.labels)

    def __getitem__(self, key):
        """
        Return the item(s) of parent array along this axis as specified by `key`.

        `key` can be any of:
            - An integer
            - A tick
            - A slice of integers or ticks
            - `numpy.newaxis`, i.e. None

        Examples
        --------

        >>> A = DataArray(np.arange(2*3*2).reshape([2,3,2]), \
                ('a', ('b', ('b1','b2','b3')), 'c'))
        >>> b = A.axes.b

        >>> np.all(b['b1'] == A[:,0,:])
        True

        >>> np.all(b['b2':] == A[:,1:,:])
        True

        >>> np.all(b['b1':'b2'] == A[:,0:1,:])
        True
        """
        # XXX We don't handle fancy indexing at the moment
        if isinstance(key, (np.ndarray, list)):
            raise NotImplementedError('We do not handle fancy indexing yet')
        parent_arr = self.parent_arr # local for speed
        parent_arr_ndim = parent_arr.ndim
        # The logic is: when using scalar indexing, the dimensionality of the
        # output is parent_arr.ndim-1, while when using slicing the output has
        # the same number of dimensions as the input.  For this reason, the
        # case when parent_arr.ndim is 1 and the indexing is scalar needs to be
        # handled separately, since the output will be 0-dimensional.  In that
        # case, we must return the plain scalar and not build a slice object
        # that would return a 1-element sub-array.
        #
        # XXX we do not here handle 0 dimensional arrays.
        # XXX fancy indexing
        if parent_arr_ndim == 1 and not isinstance(key, slice):
            sli = self.make_slice(key)
            return np.ndarray.__getitem__(parent_arr, sli)

        # For other cases (slicing or scalar indexing of ndim>1 arrays),
        # build the proper slicing object to cut into the managed array
        fullslice = self.make_slice(key)
        # now get the translated key
        key = fullslice[self.index]
        out = np.ndarray.__getitem__(parent_arr, tuple(fullslice))

        newaxes = []
        for a in parent_arr.axes:
            newaxes.append( a._copy(parent_arr=parent_arr) )

        if isinstance(key, slice):
            # we need to find the labels, if any
            if self.labels:
                newlabels = self.labels[key]
            else:
                newlabels = None
            # insert new Axis with sliced labels
            newaxis = self._copy(parent_arr=parent_arr, labels=newlabels)
            newaxes[self.index] = newaxis

        if out.ndim < parent_arr_ndim:
            # We lost a dimension, drop the axis!
            newaxes = _pull_axis(newaxes, self)

        elif out.ndim > parent_arr_ndim:
            # We were indexed by a newaxis (None),
            # need to insert an unnamed axis before this axis.
            # Do this by inserting an Axis at the end of the axes, then
            # reindexing them
            new_axis = self.__class__(None, out.ndim-1, parent_arr)
            new_ax_order = [ax.index for ax in newaxes]
            new_ax_order.insert(self.index, out.ndim-1)
            newaxes.append(new_axis)
            newaxes = _reordered_axes(newaxes, new_ax_order)

        _set_axes(out, newaxes)

        return out

    def make_slice(self, key):
        """
        Make a slicing tuple into the parent array such that
        this Axis is cut up in the requested manner

        Parameters
        ----------
        key : a slice object, single label-like item, or None
          This slice object may have arbitrary types for .start, .stop,
          in which case label labels will be looked up. The .step attribute
          of course must be None or an integer.

        Returns
        -------
        keys : parent_arr.ndim-length tuple for slicing

        """

        full_slicing = [ slice(None) ] * self.parent_arr.ndim

        # if no labels, pop in the key and pray (will raise later)
        if not self.labels:
            full_slicing[self.index] = key
            return tuple(full_slicing)

        # in either case, try to translate slicing key
        if not isinstance(key, slice):
            lookups = (key,)
        else:
            lookups = (key.start, key.stop)

        looked_up = []
        for a in lookups:
            if a is None:
                looked_up.append(a)
                continue
            try:
                idx = self._label_dict[a]
            except KeyError:
                if not isinstance(a, int):
                    raise IndexError(
                        'Could not find an index to match %s'%str(a)
                        )
                idx = a
            looked_up.append(idx)

        # if not a slice object, then pop in the translated index and return
        if not isinstance(key, slice):
            full_slicing[self.index] = looked_up[0]
            return tuple(full_slicing)

        # otherwise, go for the step size now
        step = key.step
        if not isinstance(step, (int, type(None))):
            raise IndexError(
                'Slicing step size must be an integer or None, not %s'%str(step)
                )
        looked_up = looked_up + [step]
        new_key = slice(*looked_up)
        full_slicing[self.index] = new_key
        return tuple(full_slicing)

    def at(self, label):
        """
        Return data at a given label.

        >>> narr = DataArray(np.random.standard_normal((4,5)), axes=['a', ('b', 'abcde')])
        >>> arr = narr.axes.b['c']
        >>> arr.axes
        (Axis(name='a', index=0, labels=None),)
        """
        if not self.labels:
            raise ValueError('axis must have labels to extract data at a given label')
        slicing = self.make_slice(label)
        return self.parent_arr[slicing]

    def keep(self, labels):
        """
        Keep only certain labels of an axis.

        >>> narr = DataArray(np.random.standard_normal((4,5)),
        ...                  axes=['a', ('b', 'abcde')])
        >>> arr = narr.axes.b.keep('cd')
        >>> [a.labels for a in arr.axes]
        [None, 'cd']

        >>> arr.axes.a.at('label')
        Traceback (most recent call last):
            ...
        ValueError: axis must have labels to extract data at a given label
        """

        if not self.labels:
            raise ValueError('axis must have labels to keep certain labels')

        idxs = [self._label_dict[label] for label in labels]

        parent_arr = self.parent_arr # local for speed
        parent_arr_ndim = parent_arr.ndim

        fullslice = [slice(None)] * parent_arr_ndim
        fullslice[self.index] = idxs
        out = np.ndarray.__getitem__(parent_arr, tuple(fullslice))

        # just change the current axes
        new_axes = [a._copy() for a in out.axes]
        new_axes[self.index] = self._copy(labels=labels)
        _set_axes(out, new_axes)
        return out

    def drop(self, labels):
        """
        Keep only certain labels of an axis.

        Example
        =======
        >>> darr = DataArray(np.random.standard_normal((4,5)),
        ...                  axes=['a', ('b', ['a','b','c','d','e'])])
        >>> arr1 = darr.axes.b.keep(['c','d'])
        >>> arr2 = darr.axes.b.drop(['a','b','e'])
        >>> np.all(arr1 == arr2)
        True
        """

        if not self.labels:
            raise ValueError('axis must have labels to drop labels')

        kept = [t for t in self.labels if t not in labels]
        return self.keep(kept)

    def __int__(self):
        return self.index
# -- Axis utilities ------------------------------------------------------------

def _names_to_numbers(axes, ax_ids):
    """
    Convert any axis names to axis indices. Pass through any integer ax_id,
    and convert to integer any ax_id that is an Axis.
    """
    proc_ids = []
    for ax_id in ax_ids:
        if isinstance(ax_id, basestring):
            matches = [ax for ax in axes if ax._sname == ax_id]
            if not matches:
                raise NamedAxisError('No axis named %s' % ax_id)
            proc_ids.append(matches[0].index)
        else:
            proc_ids.append(int(ax_id))
    return proc_ids

def _validate_axes(arr):
    # This should always be true our axis lists....
    assert all(i == a.index and arr is a.parent_arr
            for i,a in enumerate(arr.axes))

def _pull_axis(axes, target_axis):
    """
    Return axes removing any axis matching `target_axis`. A match
    is determined by the Axis.index
    """
    newaxes = []
    if isinstance(target_axis, (list, tuple)):
        pulled_indices = [ax.index for ax in target_axis]
    else:
        pulled_indices = [target_axis.index]
    c = 0
    for a in axes:
        if a.index not in pulled_indices:
            newaxes.append(a._copy(index=c))
            c += 1
    return newaxes

def _set_axes(dest, in_axes):
    """
    Set the axes in `dest` from `in_axes`.

    WARNING: The destination is modified in-place! The following attribute
    is added to it:

    - axes: an instance of AxesManager which manages access to axes.

    Parameters
    ----------
      dest : array
      in_axes : sequence of axis objects
    """
    # XXX: This method is called multiple times during a DataArray's lifetime.
    #      Should rethink exactly when Axis copies need to be made
    axes = []
    ax_holder = KeyStruct()
    # Create the containers for various axis-related info
    for ax in in_axes:
        new_ax = ax._copy(parent_arr=dest)
        axes.append(new_ax)
        if hasattr(ax_holder, ax._sname):
            raise NamedAxisError( """There is another Axis in this group with
                    the same name""")
        ax_holder[ax._sname] = new_ax
    # Store these containers as attributes of the destination array
    dest.axes = AxesManager(dest, axes)

def names2namedict(names):
    """Make a name map out of any name input.
    """
    raise NotImplementedError()

# -- Method Wrapping -----------------------------------------------------------

# XXX: Need to convert from positional arguments to named arguments

def _apply_reduction(opname, kwnames):
    """
    Wraps the reduction operator with name `opname`. Must supply the
    method keyword argument names, since in many cases these methods
    are called with the keyword args as positional args
    """
    super_op = getattr(np.ndarray, opname)
    if 'axis' not in kwnames:
        raise ValueError(
            'The "axis" keyword must be part of an ndarray reduction signature'
            )
    def runs_op(*args, **kwargs):
        inst = args[0]
        # re/place any additional args in the appropriate keyword arg
        for nm, val in zip(kwnames, args[1:]):
            kwargs[nm] = val
        axis = kwargs.pop('axis', None)

        if not isinstance(inst, DataArray) or axis is None:
            # do nothing special if not a DataArray, otherwise
            # this is a full reduction, so we lose all axes
            return super_op(np.asarray(inst), **kwargs)

        axes = list(inst.axes)
        # try to convert a named Axis to an integer..
        # don't try to catch an error
        axis_idx = _names_to_numbers(inst.axes, [axis])[0]
        if not kwargs.get('keepdims', False):
            axes = _pull_axis(axes, inst.axes[axis_idx])
        kwargs['axis'] = axis_idx
        arr = super_op(inst, **kwargs)
        if not is_numpy_scalar(arr):
            _set_axes(arr, axes)
        return arr
    runs_op.__name__ = opname
    runs_op.__doc__ = super_op.__doc__
    return runs_op

def is_numpy_scalar(arr):
    return arr.ndim == 0

def _apply_accumulation(opname, kwnames):
    super_op = getattr(np.ndarray, opname)
    if 'axis' not in kwnames:
        raise ValueError(
            'The "axis" keyword must be part of an ndarray reduction signature'
            )
    def runs_op(*args, **kwargs):
        inst = args[0]

        # re/place any additional args in the appropriate keyword arg
        for nm, val in zip(kwnames, args[1:]):
            kwargs[nm] = val
        axis = kwargs.pop('axis', None)
        if axis is None:
            # this will flatten the array and lose all dimensions
            return super_op(np.asarray(inst), **kwargs)

        # try to convert a named Axis to an integer..
        # don't try to catch an error
        axis_idx = _names_to_numbers(inst.axes, [axis])[0]
        kwargs['axis'] = axis_idx
        return super_op(inst, **kwargs)
    runs_op.__name__ = opname
    runs_op.__doc__ = super_op.__doc__
    return runs_op

class DataArray(np.ndarray):
    # XXX- we need to figure out where in the numpy C code .T is defined!
    @property
    def T(self):
        return self.transpose()

    def __new__(cls, data, axes=None, dtype=None, copy=False):
        # XXX if an entry of axes is a tuple, it is interpreted
        # as a (name, labels) tuple
        # Ensure the output is an array of the proper type
        arr = np.array(data, dtype=dtype, copy=copy).view(cls)

        if axes is None:
            if hasattr(data,'axes'):
                _set_axes(arr, data.axes)
                return arr
            axes = []

        elif len(axes) > arr.ndim:
            raise NamedAxisError('Axes list should have length <= array ndim')

        # Pad axes spec to match array shape
        axes = list(axes) + [None]*(arr.ndim - len(axes))

        axlist = []
        for i, axis_spec in enumerate(axes):
            if isinstance(axis_spec, basestring) or axis_spec is None:
                # string name
                name = axis_spec
                labels = None
            else:
                if len(axis_spec) != 2:
                    raise ValueError("""If the axis specification is a tuple,
                            it must be of the form (name, labels)""")
                name, labels = axis_spec
            axlist.append(Axis(name, i, arr, labels=labels))

        _set_axes(arr, axlist)
        _validate_axes(arr)

        return arr

    def set_name(self, i, name):
        self.axes[i].set_name(name)

    @property
    def names (self):
        """Returns a tuple with all the axis names."""
        return tuple((ax.name for ax in self.axes))

    def index_by(self, *args):
        return AxisIndexer(self, *args)

    def __array_finalize__(self, obj):
        """Called by ndarray on subobject (like views/slices) creation.

        Parameters
        ----------
        self : ``DataArray``
           Newly create instance of ``DataArray``
        obj : ndarray or None
           any ndarray object (if view casting)
           ``DataArray`` instance, if new-from-template
           None if triggered from DataArray.__new__ call
        """

##         print "finalizing DataArray" # dbg

        # Ref: see http://docs.scipy.org/doc/numpy/user/basics.subclassing.html

        # provide info for what's happening
##         print "finalize:\t%s\n\t\t%s" % (self.__class__, obj.__class__) # dbg
##         print "obj     :", obj.shape  # dbg
        # provide more info
        if obj is None: # own constructor, we're done
            return
        if not hasattr(obj, 'axes'): # looks like view cast
            _set_axes(self, [])
            return
        # new-from-template: we just copy the axes from the template,
        # and hope the calling rountine knows what to do with the output
##         print 'setting axes on self from obj' # dbg
        _set_axes(self, obj.axes)

        # validate the axes
        _validate_axes(self)

    def __array_prepare__(self, obj, context=None):
        "Called at the beginning of each ufunc."

##         print "preparing DataArray" # dbg

        # Ref: see http://docs.scipy.org/doc/numpy/reference/arrays.classes.html

        # provide info for what's happening
        #print "prepare:\t%s\n\t\t%s" % (self.__class__, obj.__class__) # dbg
        #print "obj     :", obj.shape  # dbg
        #print "context :", context  # dbg

        if context is not None and len(context[1]) > 1:
            "binary ufunc operation"
            other = context[1][1]
##             print "other   :", other.__class__

            if not isinstance(other,DataArray):
                return obj

##                 print "found DataArray, comparing axes"

            # walk back from the last axis on each array, check
            # that the name and shape are acceptible for broadcasting
            these_axes = list(self.axes)
            those_axes = list(other.axes)
            #print self.shape, self.names # dbg
            while these_axes and those_axes:
                that_ax = those_axes.pop(-1)
                this_ax = these_axes.pop(-1)
                # print self.shape # dbg
                this_dim = self.shape[this_ax.index]
                that_dim = other.shape[that_ax.index]
                if that_ax.name != this_ax.name:
                    # A valid name can be mis-matched IFF the other
                    # (name, length) pair is:
                    # * (None, 1)
                    # * (None, {this,that}_dim).
                    # In this case, the unnamed Axis should
                    # adopt the name of the matching Axis in the
                    # other array (handled in elsewhere)
                    if that_ax.name is not None and this_ax.name is not None:
                        raise NamedAxisError(
                            'Axis axes are incompatible for '\
                            'a binary operation: ' \
                            '%s, %s'%(self.names, other.names))
                if that_ax.labels != this_ax.labels:
                    if that_ax.labels is not None and this_ax.labels is not None:
                        raise NamedAxisError(
                            'Axis labels are incompatible for '\
                            'a binary operation.')

                # XXX: Does this dimension compatibility check happen
                #      before __array_prepare__ is even called? This
                #      error is not fired when there's a shape mismatch.
                if this_dim==1 or that_dim==1 or this_dim==that_dim:
                    continue
                raise NamedAxisError('Dimension with name %s has a '\
                                     'mis-matched shape: ' \
                                     '(%d, %d) '%(this_ax.name,
                                                  this_dim,
                                                  that_dim))
        return obj

    def __array_wrap__(self, obj, context=None):
        # provide info for what's happening
        # print "prepare:\t%s\n\t\t%s" % (self.__class__, obj.__class__) # dbg
        # print "obj     :", obj.shape  # dbg
        # print "context :", context # dbg

        other = None
        if context is not None and len(context[1]) > 1:
            "binary ufunc operation"
            other = context[1][1]
##             print "other   :", other.__class__

        if isinstance(other,DataArray):
##                 print "found DataArray, comparing names"

            # walk back from the last axis on each array to get the
            # correct names/labels
            these_axes = list(self.axes)
            those_axes = list(other.axes)
            ax_spec = []
            while these_axes and those_axes:
                this_ax = these_axes.pop(-1)
                that_ax = those_axes.pop(-1)
                # If we've broadcasted this array against another, then
                # this_ax.name may be None, in which case the new array's
                # Axis name should take on the value of that_ax
                if this_ax.name is None:
                    ax_spec.append(that_ax)
                else:
                    ax_spec.append(this_ax)
            ax_spec = ax_spec[::-1]
            # if the axes are not totally consumed on one array or the other,
            # then grab those names/labels for the rest of the dims
            if these_axes:
                ax_spec = these_axes + ax_spec
            elif those_axes:
                ax_spec = those_axes + ax_spec
        else:
            ax_spec = self.axes

        res = obj.view(type(self))
        new_axes = []
        for i, ax in enumerate(ax_spec):
            new_axes.append( ax._copy(index=i, parent_arr=res) )
        _set_axes(res, new_axes)
        return res

    def __getitem__(self, key):
        """Support x[k] access."""
        # Slicing keys:
        # * a single int
        # * a single newaxis
        # * a tuple with length <= self.ndim (may have newaxes)
        # * a tuple with length > self.ndim (MUST have newaxes)
        # * list, array, etc for fancy indexing (not implemented)

        # Cases
        if isinstance(key, list) or isinstance(key, np.ndarray):
            # fancy indexing
            # XXX need to be cast to an "ordinary" ndarray
            raise NotImplementedError
        if key is None:
            key = (key,)

        if isinstance(key, tuple):
            old_shape = self.shape
            old_axes = self.axes
            new_shape, new_axes, key = _make_singleton_axes(self, key)
            # Will undo this later
            self.shape = new_shape
            _set_axes(self, new_axes)

            # Pop the axes off in descending order to prevent index renumbering
            # headaches.
            reductions = list(reversed(sorted(zip(key, new_axes),
                                              key=lambda t: t[1].index)))
            arr = self
            for k,ax in reductions:
                arr = arr.axes[ax.index][k]

            # restore old shape and axes
            self.shape = old_shape
            _set_axes(self, old_axes)
        else:
            arr = self.axes[0][key]

        return arr

    def __str_repr_helper(self, ary_repr):
        """Helper function for __str__ and __repr__. Produce a text
        representation of the axis suitable for eval() as an argument to a
        DataArray constructor."""
        axis_spec = repr(tuple(ax.name if ax.labels is None
            else (ax.name, tuple(ax.labels)) for ax in self.axes))
        return "%s(%s,\n%s)" % \
                (self.__class__.__name__, ary_repr, axis_spec)

    def __str__(self):
        return self.__str_repr_helper(np.asarray(self).__str__())

    def __repr__(self):
        return self.__str_repr_helper(np.asarray(self).__repr__())

    # Methods from ndarray

    def transpose(self, *axes):
        # implement tuple-or-*args logic of np.transpose
        axes = list(axes)
        if not axes:
            axes = list(range(self.ndim-1,-1,-1))
        # expand sequence if sequence passed as first and only arg
        elif len(axes) < self.ndim:
            try:
                axes = list(axes[0])
            except TypeError:
                pass
        proc_axids = _names_to_numbers(self.axes, axes)
        out = np.ndarray.transpose(self, proc_axids)
        _set_axes(out, _reordered_axes(self.axes, proc_axids, parent=out))
        return out
    transpose.__doc__ = np.ndarray.transpose.__doc__

    def swapaxes(self, axis1, axis2):
        # form a transpose operation with axes specified
        # by (axis1, axis2) swapped
        axis1, axis2 = _names_to_numbers(self.axes, [axis1, axis2])
        ax_idx = list(range(self.ndim))
        tmp = ax_idx[axis1]
        ax_idx[axis1] = ax_idx[axis2]
        ax_idx[axis2] = tmp
        out = np.ndarray.transpose(self, ax_idx)
        _set_axes(out, _reordered_axes(self.axes, ax_idx, parent=out))
        return out
    swapaxes.__doc__ = np.ndarray.swapaxes.__doc__

    def ptp(self, axis=None, out=None):
        mn = self.min(axis=axis)
        mx = self.max(axis=axis, out=out)
        if isinstance(mn, np.ndarray):
            mx -= mn
            return mx
        else:
            return mx-mn
    ptp.__doc__ = np.ndarray.ptp.__doc__

    # -- Various extraction and reshaping methods ----------------------------
    def diagonal(self, *args, **kwargs):
        # reverts to being an ndarray
        args = (np.asarray(self),) + args
        return np.diagonal(*args, **kwargs)
    diagonal.__doc__ = np.ndarray.diagonal.__doc__

    def flatten(self, **kwargs):
        # reverts to being an ndarray
        return np.asarray(self).flatten(**kwargs)
    flatten.__doc__ = np.ndarray.flatten.__doc__

    def ravel(self, **kwargs):
        # reverts to being an ndarray
        return np.asarray(self).ravel(**kwargs)
    ravel.__doc__ = np.ndarray.ravel.__doc__

    def repeat(self, *args, **kwargs):
        raise NotImplementedError

    def squeeze(self):
        axes = list(self.axes)
        pinched_axes = [x for x in axes if self.shape[x.index] == 1]
        squeezed_shape = [d for d in self.shape if d > 1]
        axes = _pull_axis(axes, pinched_axes)
        arr = self.reshape(squeezed_shape)
        _set_axes(arr, axes)
        return arr

    def reshape(self, *args, **kwargs):
        # XXX:
        # * reshapes such as a.reshape(a.shape + (1,)) will be supported
        # * reshapes such as a.ravel() will return ndarray
        # * reshapes such as a.reshape(x', y', z') ???
        # print 'reshape called', args, kwargs # dbg
        if len(args) == 1:
            if isinstance(args[0], (tuple, list)):
                args = args[0]
            else:
                return np.asarray(self).reshape(*args)
        # if adding/removing length-1 dimensions, then add an unnamed Axis
        # or pop an Axis
        old_shape = list(self.shape)
        new_shape = list(args)
        old_non_single_dims = [d for d in old_shape if d > 1]
        new_non_single_dims = [d for d in new_shape if d > 1]
        axes_to_pull = []
        axes = list(self.axes)
        if old_non_single_dims == new_non_single_dims:
            # pull axes first
            i = j = 0
            while i < len(new_shape) and j < len(old_shape):
                if new_shape[i] != old_shape[j] and old_shape[j] == 1:
                    axes_to_pull.append(self.axes[j])
                else:
                    i += 1
                j += 1
            # pull anything that extends past the length of the new shape
            axes_to_pull += [self.axes[i] for i in range(j, len(old_shape))]
            old_shape = [self.shape[ax.index]
                         for ax in axes if ax not in axes_to_pull]
            axes = _pull_axis(axes, axes_to_pull)
            # now append axes
            i = j = 0
            axes_order = []
            while i < len(new_shape) and j < len(old_shape):
                if new_shape[i] != old_shape[j] and new_shape[i] == 1:
                    idx = len(axes)
                    axes.append( Axis(None, idx, self) )
                    axes_order.append(idx)
                else:
                    axes_order.append(j)
                    j += 1
                i += 1
            # append None axes for all shapes past the length of the old shape
            new_idx = range(i, len(new_shape))
            axes += [Axis(None, idx, self) for idx in new_idx]
            axes_order += new_idx
            axes = _reordered_axes(axes, axes_order)
            arr = super(DataArray, self).reshape(*new_shape)
            _set_axes(arr, axes)
            return arr

        # if dimension sizes can be moved around between existing axes,
        # then go ahead and try to keep the Axis meta-data
        raise NotImplementedError

    # -- Sorting Ops ---------------------------------------------------------
    # ndarray sort with axis==None flattens the array: return ndarray

    # Otherwise, if there are labels at the axis in question, then
    # the sample-to-label correspondence becomes inconsistent across
    # the remaining axes. Also return a plain ndarray.

    # Otherwise, order the axis in question--default axis is -1

    # XXX: Might be best to always return ndarray, since the return
    #      type is so inconsistent
    def sort(self, **kwargs):
        axis = kwargs.get('axis', -1)
        if axis is not None:
            axis = _names_to_numbers(self.axes, [axis])[0]
            kwargs['axis'] = axis
        if axis is None or self.axes[axis].labels:
            # Returning NEW ndarray
            arr = np.asarray(self).copy()
            arr.sort(**kwargs)
            return arr
        # otherwise, just do the op on this array
        super(DataArray, self).sort(**kwargs)

    def argsort(self, **kwargs):
        axis = kwargs.get('axis', -1)
        if axis is not None:
            axis = _names_to_numbers(self.axes, [axis])[0]
            kwargs['axis'] = axis
        if axis is None or self.axes[axis].labels:
            # Returning NEW ndarray
            arr = np.asarray(self)
            return arr.argsort(**kwargs)
        # otherwise, just do the op on this array
        axes = list(self.axes)
        arr = super(DataArray, self).argsort(**kwargs)
        _set_axes(arr, axes)
        return arr

    # -- Reductions ----------------------------------------------------------
    mean = _apply_reduction('mean', ('axis', 'dtype', 'out', 'keepdims'))
    var = _apply_reduction('var', ('axis', 'dtype', 'out', 'ddof', 'keepdims'))

    def std(self, *args, **kwargs):
        ret = self.var(*args, **kwargs)
        if isinstance(ret, np.ndarray):
            ret = np.sqrt(ret, out=ret)
        elif hasattr(ret, 'dtype'):
            ret = ret.dtype.type(np.sqrt(ret))
        else:
            ret = np.sqrt(ret)
        return ret
    std.__doc__ = np.ndarray.std.__doc__

    min = _apply_reduction('min', ('axis', 'out', 'keepdims'))
    max = _apply_reduction('max', ('axis', 'out', 'keepdims'))

    sum = _apply_reduction('sum', ('axis', 'dtype', 'out', 'keepdims'))
    prod = _apply_reduction('prod', ('axis', 'dtype', 'out', 'keepdims'))

    all = _apply_reduction('all', ('axis', 'dtype', 'out', 'keepdims'))
    any = _apply_reduction('any', ('axis', 'dtype', 'out', 'keepdims'))

    ### these change the meaning of the axes..
    ### should probably return ndarrays
    argmax = _apply_reduction('argmax', ('axis', 'out'))
    argmin = _apply_reduction('argmin', ('axis', 'out'))

    # -- Accumulations -------------------------------------------------------
    cumsum = _apply_accumulation('cumsum', ('axis', 'dtype', 'out'))
    cumprod = _apply_accumulation('cumprod', ('axis', 'dtype', 'out'))

# -- DataArray utilities -------------------------------------------------------

def _reordered_axes(axes, axis_indices, parent=None):
    ''' Perform axis reordering according to `axis_indices`
    Checks to ensure that all axes have the same parent array.
    Parameters
    ----------
    axes : sequence of axes
       The axis indices in this list reflect the axis ordering before
       the permutation given by `axis_indices`
    axis_indices : sequence of ints
       indices giving new order of axis numbers
    parent : ndarray or None
       if not None, used as parent for all created axes

    Returns
    -------
    ro_axes : sequence of axes
       sequence of axes (with the same parent array)
       in arbitrary order with axis indices reflecting
       reordering given by `axis_indices`

    Examples
    --------
    >>> a = Axis('x', 0, None)
    >>> b = Axis('y', 1, None)
    >>> c = Axis(None, 2, None)
    >>> res = _reordered_axes([a,b,c], (1,2,0))
    '''

    new_axes = []
    for new_ind, old_ind in enumerate(axis_indices):
        ax = axes[old_ind]
        if parent is None:
            parent_arr = ax.parent_arr
        else:
            parent_arr = parent
        new_ax = ax._copy(index=new_ind, parent_arr=parent_arr)
        new_axes.append(new_ax)
    return new_axes

def _expand_ellipsis(key, ndim):
    "Expand the slicing tuple if the Ellipsis object is present."
    # Ellipsis can only occur once (not totally the same as NumPy),
    # which apparently allows multiple Ellipses to follow one another
    kl = list(key)
    ecount = kl.count(Ellipsis)
    if ecount > 1:
        raise IndexError('invalid index')
    if ecount < 1:
        return key
    e_index = kl.index(Ellipsis)
    kl_end = kl[e_index+1:] if e_index < len(key)-1 else []
    kl_beg = kl[:e_index]
    kl_middle = [slice(None)] * (ndim - len(kl_end) - len(kl_beg))
    return tuple( kl_beg + kl_middle + kl_end )

def _make_singleton_axes(arr, key):
    """
    Parse the slicing key to determine whether the array should be
    padded with singleton dimensions prior to slicing. Also expands
    any Ellipses in the slicing key.

    Parameters
    ----------
    arr : DataArray
    key : slicing tuple

    Returns
    -------
    (shape, axes, key)

    These are the new shape, with singleton axes included; the new axes,
    with an unnamed Axis at each singleton dimension; and the new
    slicing key, with `newaxis` keys replaced by slice(None)
    """

    key = _expand_ellipsis(key, arr.ndim)
    if len(key) <= arr.ndim and None not in key:
        return arr.shape, arr.axes, key

    # The full slicer will be length=arr.ndim + # of dummy-dims..
    # Boost up the slices to full "rank" ( can cut it down later for savings )
    n_new_dims = len([x for x in key if x is None])
    key = key + (slice(None),) * (arr.ndim + n_new_dims - len(key))
    # wherever there is a None in the key,
    # * replace it with slice(None)
    # * place a new dimension with length 1 in the shape,
    # * and add a new unnamed Axis to the axes
    new_dims = []
    new_key = []
    d_cnt = 0
    new_ax_pos = arr.ndim
    new_axes = list(arr.axes)
    ax_order = []
    for k in key:
        if k is None:
            new_key.append(slice(None))
            new_dims.append(1)
            # add a new Axis at the end of the list, then reorder
            # the list later to ensure the Axis indices are accurate
            new_axes.append(Axis(None, new_ax_pos, arr))
            ax_order.append(new_ax_pos)
            new_ax_pos += 1
        else:
            new_key.append(k)
            try:
                new_dims.append(arr.shape[d_cnt])
                ax_order.append(d_cnt)
                d_cnt += 1
            except IndexError:
                raise IndexError('too many indices')
    ro_axes = _reordered_axes(new_axes, ax_order)
    # Cut down all trailing "slice(None)" objects at the end of the new key.
    # (But! it seems we have to leave in at least one slicing element
    #  in order to get a new array)
    while len(new_key)>1 and new_key[-1] == slice(None):
        new_key.pop()
    return tuple(new_dims), ro_axes, tuple(new_key)

if __name__ == "__main__":
    import doctest
    doctest.testmod()


# flake8: noqa
# pylint: disable=invalid-name
# pylint: disable=line-too-long
# pylint: disable=missing-docstring
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
"""
Most Cited Documents
===============================================================================

>>> from techminer2.analyze.citation import most_cited_documents
>>> documents = most_cited_documents(
...     #
...     # FUNCTION PARAMS:
...     metric="global_citations",
...     top_n=20,
...     #
...     # CHART PARAMS:
...     title=None,
...     field_label=None,
...     metric_label=None,
...     textfont_size=10,
...     marker_size=7,
...     line_width=1.5,
...     yshift=4,
...     #
...     # DATABASE PARAMS:
...     root_dir="example/",
...     database="main",
...     year_filter=(None, None),
...     cited_by_filter=(None, None),
... )
--INFO-- The file 'data/regtech/reports/most_global_cited_documents__abstracts.txt' was created.
--INFO-- The file 'data/regtech/reports/most_global_cited_documents__prompt.txt' was created.

>>> documents.fig_.write_html("sphinx/_static/analyze/citation/most_global_cited_documents.html")

.. raw:: html

    <iframe src="../../../../_static/analyze/citation/most_global_cited_documents.html"
    height="600px" width="100%" frameBorder="0"></iframe>

>>> print(documents.df_.to_markdown())
| article                                                       |   year |   rank_gcs |   global_citations |   rank_lcs |   local_citations |   global_citations_per_year |   local_citations_per_year | doi                                |
|:--------------------------------------------------------------|-------:|-----------:|-------------------:|-----------:|------------------:|----------------------------:|---------------------------:|:-----------------------------------|
| Anagnostopoulos I, 2018, J ECON BUS, V100, P7                 |   2018 |          1 |                153 |          1 |                17 |                      25.5   |                      2.833 | 10.1016/J.JECONBUS.2018.07.003     |
| Arner DW, 2017, NORTHWEST J INTL LAW BUS, V37, P373           |   2017 |          2 |                150 |          2 |                16 |                      21.429 |                      2.286 | nan                                |
| Butler T, 2019, PALGRAVE STUD DIGIT BUS ENABL, P85            |   2019 |          3 |                 33 |          3 |                14 |                       6.6   |                      2.8   | 10.1007/978-3-030-02330-0_6        |
| Baxter LG, 2016, DUKE LAW J, V66, P567                        |   2016 |          4 |                 30 |          4 |                 8 |                       3.75  |                      1     | nan                                |
| Buckley RP, 2020, J BANK REGUL, V21, P26                      |   2020 |          5 |                 24 |          6 |                 5 |                       6     |                      1.25  | 10.1057/S41261-019-00104-1         |
| Kavassalis P, 2018, J RISK FINANC, V19, P39                   |   2018 |          6 |                 21 |          5 |                 8 |                       3.5   |                      1.333 | 10.1108/JRF-07-2017-0111           |
| Singh C, 2020, J MONEY LAUND CONTROL, V24, P464               |   2020 |          7 |                 14 |          9 |                 3 |                       3.5   |                      0.75  | 10.1108/JMLC-09-2020-0100          |
| Muganyi T, 2022, FINANCIAL INNOV, V8                          |   2022 |          8 |                 13 |         19 |                 1 |                       6.5   |                      0.5   | 10.1186/S40854-021-00313-6         |
| Ryan P, 2020, ICEIS - PROC INT CONF ENTERP , V2, P787         |   2020 |          9 |                 12 |         10 |                 3 |                       3     |                      0.75  | nan                                |
| von Solms J, 2021, J BANK REGUL, V22, P152                    |   2021 |         10 |                 11 |          8 |                 4 |                       3.667 |                      1.333 | 10.1057/S41261-020-00134-0         |
| Arner DW, 2017, HANDBBLOCKCHAIN, DIGIT FINANC, P359           |   2017 |         11 |                 11 |         11 |                 3 |                       1.571 |                      0.429 | 10.1016/B978-0-12-810441-5.00016-6 |
| Turki M, 2020, HELIYON, V6                                    |   2020 |         12 |                 11 |         16 |                 2 |                       2.75  |                      0.5   | 10.1016/J.HELIYON.2020.E04949      |
| Kurum E, 2020, J FINANC CRIME                                 |   2020 |         13 |                 10 |         12 |                 3 |                       2.5   |                      0.75  | 10.1108/JFC-04-2020-0051           |
| Butler T, 2018, J RISK MANG FINANCIAL INST, V11, P19          |   2018 |         14 |                  8 |          7 |                 5 |                       1.333 |                      0.833 | nan                                |
| Turki M, 2021, ADV INTELL SYS COMPUT, V1141, P349             |   2021 |         15 |                  7 |         20 |                 1 |                       2.333 |                      0.333 | 10.1007/978-981-15-3383-9_32       |
| Becker M, 2020, INTELL SYST ACCOUNT FINANCE M, V27, P161      |   2020 |         17 |                  5 |         22 |                 1 |                       1.25  |                      0.25  | 10.1002/ISAF.1479                  |
| Das SR, 2019, J FINANCIAL DATA SCI, V1, P8                    |   2019 |         16 |                  5 |         21 |                 1 |                       1     |                      0.2   | 10.3905/JFDS.2019.1.2.008          |
| Waye V, 2020, ADELAIDE LAW REV, V40, P363                     |   2020 |         18 |                  5 |         23 |                 1 |                       1.25  |                      0.25  | nan                                |
| Pantielieieva N, 2020, LECTURE NOTES DATA ENG COMMUN, V42, P1 |   2020 |         20 |                  4 |         28 |                 0 |                       1     |                      0     | 10.1007/978-3-030-35649-1_1        |
| Brand V, 2020, UNIV NEW SOUTH WALES LAW J, V43, P801          |   2020 |         19 |                  4 |         13 |                 3 |                       1     |                      0.75  | nan                                |

"""

import os
from dataclasses import dataclass

import plotly.express as px

from ..._common._read_records import read_records
from ..._common.format_prompt_for_records import format_prompt_for_records
from ..._common.format_report_for_records import format_report_for_records

MARKER_COLOR = "#7793a5"
MARKER_LINE_COLOR = "#465c6b"

def most_cited_documents(
    #
    # PARAMS:
    metric,
    top_n,
    #
    # CHART PARAMS:
    title=None,
    field_label=None,
    metric_label=None,
    textfont_size=10,
    marker_size=7,
    line_width=1.5,
    yshift=4,
    #
    # DATABASE PARAMS:
    root_dir: str = "./",
    database: str = "main",
    year_filter: tuple = (None, None),
    cited_by_filter: tuple = (None, None),
    **filters,
):
    """
    :meta private:
    """

    # -----------------------------------------------------------------------------------
    #
    def compute_metrics(
        #
        # PARAMS:
        metric,
        top_n,
        #
        # DATABASE PARAMS:
        root_dir,
        database,
        year_filter,
        cited_by_filter,
        **filters,
    ):
        data_frame = read_records(
            root_dir=root_dir,
            database=database,
            year_filter=year_filter,
            cited_by_filter=cited_by_filter,
            **filters,
        )

        max_year = data_frame.year.dropna().max()

        #
        # Global citations per year
        data_frame["global_citations"] = data_frame.global_citations.astype(int)
        data_frame = data_frame.assign(
            global_citations_per_year=data_frame.global_citations.astype(float)
            / (max_year - data_frame.year + 1)
        )
        data_frame = data_frame.assign(
            global_citations_per_year=data_frame.global_citations_per_year.round(3)
        )

        #
        # Global citation score rank
        data_frame = data_frame.sort_values(
            ["global_citations", "local_citations", "year", "authors"],
            ascending=[False, False, True, True],
        )
        data_frame["rank_gcs"] = range(1, len(data_frame) + 1)

        #
        # Local citation score rank
        data_frame = data_frame.sort_values(
            ["local_citations", "global_citations", "year", "authors"],
            ascending=[False, False, True, True],
        )
        data_frame["rank_lcs"] = range(1, len(data_frame) + 1)

        #
        # Local citations per year
        data_frame["local_citations"] = data_frame.local_citations.astype(int)
        data_frame = data_frame.assign(
            local_citations_per_year=data_frame.local_citations.astype(float)
            / (max_year - data_frame.year + 1)
        )
        data_frame = data_frame.assign(
            local_citations_per_year=data_frame.local_citations_per_year.round(3)
        )

        #
        # Set the index to the article identifier
        data_frame = data_frame.set_index("article")

        #
        # Selects the top-n documents
        data_frame = data_frame.sort_values([metric], ascending=False).head(top_n)

        return data_frame

    # -----------------------------------------------------------------------------------
    # Create the figure
    #
    def create_fig(
        data_frame,
        metric,
        #
        # CHART PARAMS:
        title,
        field_label,
        metric_label,
        textfont_size,
        marker_size,
        line_width,
        yshift,
    ):
        data_frame = data_frame.copy()

        metric_label = metric.replace("_", " ").upper() if metric_label is None else metric_label

        field_label = (
            metric.replace("_", " ").upper() + " RANKING" if field_label is None else field_label
        )

        data_frame["Rank"] = list(range(1, len(data_frame) + 1))

        fig = px.line(
            data_frame,
            x="Rank",
            y=metric,
            hover_data=data_frame.columns.to_list(),
            markers=True,
        )

        fig.update_traces(
            marker={
                "size": marker_size,
                "line": {"color": MARKER_LINE_COLOR, "width": 1},
            },
            marker_color=MARKER_COLOR,
            line={"color": MARKER_LINE_COLOR, "width": line_width},
        )
        fig.update_layout(
            paper_bgcolor="white",
            plot_bgcolor="white",
            title_text=title if title is not None else "",
        )
        fig.update_yaxes(
            linecolor="gray",
            linewidth=1,
            gridcolor="lightgray",
            griddash="dot",
            title=metric_label,
        )
        fig.update_xaxes(
            linecolor="gray",
            linewidth=1,
            gridcolor="lightgray",
            griddash="dot",
            title=field_label,
        )

        for name, row in data_frame.iterrows():
            fig.add_annotation(
                x=row["Rank"],
                y=row[metric],
                text=name,
                showarrow=False,
                textangle=-90,
                yanchor="bottom",
                font={"size": textfont_size},
                yshift=yshift,
            )

        return fig

    # -----------------------------------------------------------------------------------
    #
    def create_report(
        data_frame,
        metric,
    ):
        report_filename = "most_"
        if metric == "local_citations":
            report_filename += "local_cited_"
        if metric == "global_citations":
            report_filename += "global_cited_"
        if database == "main":
            report_filename += "documents__abstracts.txt"
        if database == "references":
            report_filename += "references__abstracts.txt"

        format_report_for_records(
            root_dir=root_dir,
            target_dir="",
            records=data_frame,
            report_filename=report_filename,
        )

    # -----------------------------------------------------------------------------------
    #
    def create_prompt(
        data_frame,
        metric,
    ):
        prompt_filename = "most_"
        if metric == "local_citations":
            prompt_filename += "local_cited_"
        if metric == "global_citations":
            prompt_filename += "global_cited_"
        if database == "main":
            prompt_filename += "documents__prompt.txt"
        if database == "references":
            prompt_filename += "references__prompt.txt"

        main_text = (
            "You are researcher writing a scientific paper. "
            "Your task is to write a short summary for each of the following "
            "paragraphs. Summarize each one the paragraphs below, delimited by "
            "triple backticks, in at most 30 words, focusing on the main "
            "contribution described in the paragraph. "
        )

        data_frame = data_frame.copy()
        data_frame["article"] = data_frame.index
        text = format_prompt_for_records(
            main_text,
            data_frame,
            weight=None,
        )

        file_name = os.path.join(root_dir, "reports", prompt_filename)
        with open(file_name, "w", encoding="utf-8") as file:
            print(text, file=file)

        print(f"--INFO-- The file '{file_name}' was created.")

    #
    #
    # MAIN CODE:
    #
    #
    data_frame = compute_metrics(
        #
        # PARAMS:
        metric=metric,
        top_n=top_n,
        #
        # DATABASE PARAMS:
        root_dir=root_dir,
        database=database,
        year_filter=year_filter,
        cited_by_filter=cited_by_filter,
        **filters,
    )

    #
    # Abstracts Report:
    create_report(
        data_frame=data_frame,
        metric=metric,
    )

    #
    # Creates ChatGPT Prompt
    create_prompt(
        data_frame=data_frame,
        metric=metric,
    )

    #
    # Selects the columns
    data_frame = data_frame[
        [
            "year",
            "rank_gcs",
            "global_citations",
            "rank_lcs",
            "local_citations",
            "global_citations_per_year",
            "local_citations_per_year",
            "doi",
        ]
    ]

    fig = create_fig(
        data_frame=data_frame,
        metric=metric,
        #
        # CHART PARAMS:
        title=title,
        field_label=field_label,
        metric_label=metric_label,
        textfont_size=textfont_size,
        marker_size=marker_size,
        line_width=line_width,
        yshift=yshift,
    )

    @dataclass
    class Results:
        df_ = data_frame
        fig_ = fig

    return Results()


# -*- coding: utf-8 -*-
# @Time : 2020/4/16 20:59
# @Author : Zhongyi Hua
# @FileName: correct2.py
# @Usage: correct annotation error using CPGAVAS2 and PGA result.
# @Note:
# @E-mail: njbxhzy@hotmail.com

from pathlib import Path
from collections import defaultdict
from datetime import date

from Bio import SeqIO
from Bio.Blast.Applications import NcbiblastnCommandline, NcbimakeblastdbCommandline
from utilities import Check, Correct, CurateSequence, add_rps12, read_geseq, read_anno

# !!NOTICE!! All methods in this scirpt can only be used in a loop because they are under continue decorator.
#@continue_on_error
# This is the old version !!
def isomerism_old(asm_path1:Path, asm_path2:Path, ref_path: Path, tmp_dir:Path):
    """_summary_

    Args:
        asm_path1 (Path): _description_
        asm_path2 (Path): _description_
        ref_path (Path): _description_
        tmpdir (Path): A temporary directory

    Returns:
        isomer (Path): The selected isomer path
    """
    path1_seq = SeqIO.read(asm_path1, 'fasta')
    path2_seq = SeqIO.read(asm_path2, 'fasta')
    ref_seq = SeqIO.read(ref_path, 'fasta')
    asm_dict = {path1_seq.id: asm_path1,
                path2_seq.id: asm_path2,
                ref_seq.id:   ref_path}
    query_path = (Path(__file__) / '../ref/ndh.fa').resolve()
    subject_path = tmp_dir / 'subject.fasta'

    SeqIO.write([path1_seq, path2_seq, ref_seq], subject_path, 'fasta')

    dbcmd = NcbimakeblastdbCommandline(input_file=subject_path, dbtype='nucl')
    qcmd =  NcbiblastnCommandline(
        query=query_path,
        db=subject_path,
        evalue=0.001,
        out=tmp_dir / 'blastn.res',
        outfmt="6 qseqid sseqid sstrand evalue")

    dbcmd()
    qcmd()

    #{seqid1: 0, seqid2: 0, refid: 0}
    # if one gene on plus strand, add 1, otherwise minus 1
    tmpstrand_dict = defaultdict(int)
    for _line in Path(tmp_dir / 'blastn.res').read_text(encoding='utf-8').strip().split('\n'):
        _tmp_eles = _line.split('\t')
        if _tmp_eles[1] == 'plus':
            tmpstrand_dict[_tmp_eles[1]] += 1
        tmpstrand_dict[_tmp_eles[1]] -= 1
    #seqid1strand, seqid2strand, refstrand
    if tmpstrand_dict.get(path1_seq.id) == tmpstrand_dict.get(ref_seq.id):
        print(asm_path1)
        return asm_dict.get(path1_seq.id)
    if tmpstrand_dict.get(path2_seq.id) == tmpstrand_dict.get(ref_seq.id):
        print(asm_path2)
        return asm_dict.get(path2_seq.id)
    print('Please check', asm_path1, 'manually')
    return None

def isomerism(fasta_path:Path, ref_path: Path, tmp_dir:Path):
    """Select the right path from the chloroplast paths

    Args:
        fasta_path (Path): The input fasta path
        ref_path (Path): A
        tmp_dir (Path): A temporary directory path

    Returns:
         selected_path (Bio.SeqRecord): The selected sequence
    """
    main_ins = CurateSequence(fasta_path, tmp_dir)
    selected_path = main_ins.iso(ref_path)
    return selected_path

#@continue_on_error
def curate(fasta_path: Path, tmp_dir: Path):
    """ Check and curate sequence before annotation

    Args:
        fasta_path (Path): The input fasta path
        tmp_dir (Path): A temporary directory path

    Returns:
        seq_fixed (Bio.Seq.Seq): The fixed sequence
    """
    main_ins = CurateSequence(fasta_path, tmp_dir)
    seq_fixed = main_ins.curate()
    return seq_fixed

#@continue_on_error
def tidy_geseq(geseq_gff_path: Path):
    """Tidy gff output

    1. Remove "note" and "gbkey" in GeSeq gff file qualifiers.
    2. Add CDS and tRNA to correct gene parent

    Args:
        geseq_gff_path (Path): _description_
    """
    raw_record = read_geseq(geseq_gff_path)
    return raw_record

#@continue_on_error
def correct(gb_path1:Path, gb_path2: Path, prefix: str):
    """Correct chloroplast genome annotation using results from two softwares

    Args:
        gb_path1 (Path): The path of the GenBank file generated by a more reliable software (e.g. ).
        gb_path2 (Path): The path of the GenBank file generated by annother software (e.g. CPGAVAS2).

    Return:
        renumbered_record (SeqRecord): A SeqRecord object with CoreSeqFeature subfeatures
    """
    record1 = read_anno(gb_path1, 'gb')
    record2 = read_anno(gb_path2, 'gb')
    correct_ins = Correct(record1, record2, record1.seq)
    corrected_record = correct_ins.correct()
    # Set id for check report
    corrected_record.id = record1.id
    # Check corrected gene
    check_ins = Check(corrected_record)
    # Renumber first to ensure the gene id in check report is in line with the result
    total_gene_num = check_ins.renumber(prefix)
    check_ins.check_name()
    check_ins.check_region()
    check_ins.check_cds(record1.seq, add_pseudo=True)
    renumbered_record = check_ins.record
    # Add rps12
    rps12_lst = add_rps12(gb_path1, prefix, total_gene_num+1)
    renumbered_record.features += rps12_lst
    # For safe, return a SeqRecord with annotations
    renumbered_record.annotations = {
            'molecule_type': 'DNA',
            'topology': 'circular',
            'data_file_division': 'PLN',
            'date': date.today().strftime('%d-%b-%Y'),
            'accessions': [renumbered_record.id],
            'sequence_version': 1,
            'keywords': [''],
        }
    return renumbered_record

#@continue_on_error
def check(input_path: Path, seq_path=None):
    """Check annotatiosn validity

    Args:
        input_path (Path): The GenBank / GFF file inputs.
        seq_path (path, optional): If check the gff file, a fasta sequence is necessary
    """
    record1 = read_anno(input_path, 'gb')
    if seq_path:
        seq_ins = SeqIO.read(seq_path, 'fasta').seq
    else:
        seq_ins = record1.seq
    tmp_check = Check(record1)
    tmp_check.check_cds(seq_ins)
    tmp_check.check_name()
    tmp_check.check_region()

#@continue_on_error
def convert(input_path: Path, ftype):
    """Convert file input to a SeqRecord with CoreSeqFeature

    Args:
        input_path (Path): Input file path
        ftype (str): File type

    Returns:
        record (SeqRecord):  A SeqRecord with CoreSeqFeature
    """
    record = read_anno(input_path, ftype)
    return record


import argparse
from decimal import Decimal
from typing import Dict, Type
import os
import sys
import pathlib

import psutil

from archai.common.config import Config
from archai.common import utils

def _create_ram_disk(req_ram:int, path:str)->bool:
    os.makedirs(path, exist_ok=True)
    return True

    # tmp_filepath = os.path.join(path,'delete_me.temp')
    # disk_speed_command = f'dd if=/dev/zero of="{tmp_filepath}" bs=4k count=100000; rm "{tmp_filepath}"'
    # utils.exec_shell_command(disk_speed_command)

    # avail_mem = psutil.virtual_memory().available
    # print(f'RAM Disk params: req_ram={req_ram}, avail_mem={avail_mem}, path={path}')
    # if avail_mem > req_ram:
    #     utils.exec_shell_command(f'sudo mount -t tmpfs -o size={req_ram} pt_data "{path}"')
    #     utils.exec_shell_command(f'sudo mount') # display mounts
    #     utils.exec_shell_command(disk_speed_command)
    #     return True
    # else:
    #     print('RAM disk is not created because not enough memory')
    #     return False

def main():
    parser = argparse.ArgumentParser(description='Archai data install')
    parser.add_argument('--dataroot', type=str, default='~/dataroot',
                        help='path to dataroot on local drive')
    parser.add_argument('--dataset', type=str, default='cifar10',
                        help='Name of the dataset for which confs/dataset/name.yaml should exist and have name of folder or tar file it resides in')
    args, extra_args = parser.parse_known_args()

    pt_data_dir = os.environ.get('PT_DATA_DIR', '')
    if not pt_data_dir:
        raise RuntimeError('This script needs PT_DATA_DIR environment variable with path to dataroot on cloud drive')
    pt_data_dir = utils.full_path(pt_data_dir)
    print('pt_data_dir:', pt_data_dir)

    conf_data_filepath = f'confs/datasets/{args.dataset}.yaml'
    print('conf_data_filepath:', conf_data_filepath)

    conf_data = Config(config_filepath=conf_data_filepath)['dataset']
    storage_name = conf_data['storage_name']
    tar_filepath = os.path.join(pt_data_dir, storage_name + '.tar')
    if not os.path.isfile(tar_filepath):
        raise RuntimeError(f'Tar file for dataset at {tar_filepath} was not found')

    tar_size = pathlib.Path(tar_filepath).stat().st_size
    print('tar_filepath:', tar_filepath, 'tar_size:', tar_size)

    local_dataroot = utils.full_path(args.dataroot)
    print('local_dataroot:', local_dataroot)
    _create_ram_disk(tar_size, local_dataroot)
    # os.makedirs(local_dataroot, exist_ok=True)

    utils.exec_shell_command(f'tar -xf "{tar_filepath}" -C "{local_dataroot}"')

    print(f'dataset copied from {tar_filepath} to {local_dataroot} sucessfully')

if __name__ == '__main__':
    main()


import unittest
from unittest import mock
from requests.exceptions import ConnectionError

from betfairlightweight import APIClient
from betfairlightweight.endpoints.keepalive import KeepAlive, APIError
from betfairlightweight.exceptions import KeepAliveError, InvalidResponse
from betfairlightweight.resources.authresources import KeepAliveResource
from tests.tools import create_mock_json

class KeepAliveTest(unittest.TestCase):
    def setUp(self):
        client = APIClient("username", "password", "app_key", "UK")
        self.keep_alive = KeepAlive(client)

    @mock.patch("betfairlightweight.endpoints.keepalive.KeepAlive.request")
    def test_call(self, mock_response):
        mock = create_mock_json("tests/resources/keep_alive_success.json")
        mock_response.return_value = (mock.Mock(), mock.json(), 1.3)
        response = self.keep_alive()

        assert isinstance(response, KeepAliveResource)
        assert self.keep_alive.client.session_token == mock.json().get("token")

    @mock.patch("betfairlightweight.baseclient.BaseClient.keep_alive_headers")
    @mock.patch("betfairlightweight.baseclient.requests.post")
    def test_request(self, mock_post, mock_keep_alive_headers):
        mock_response = create_mock_json("tests/resources/logout_success.json")
        mock_post.return_value = mock_response

        url = "https://identitysso.betfair.com/api/keepAlive"
        response = self.keep_alive.request()

        mock_post.assert_called_once_with(
            url, headers=mock_keep_alive_headers, timeout=(3.05, 16)
        )
        assert response[1] == mock_response.json()

    @mock.patch("betfairlightweight.baseclient.BaseClient.keep_alive_headers")
    @mock.patch("betfairlightweight.baseclient.requests.post")
    def test_request_error(self, mock_post, mock_keep_alive_headers):
        mock_post.side_effect = ValueError()

        with self.assertRaises(APIError):
            self.keep_alive.request()

        mock_post.side_effect = ConnectionError()

        with self.assertRaises(APIError):
            self.keep_alive.request()

    @mock.patch(
        "betfairlightweight.endpoints.keepalive.json.loads", side_effect=ValueError
    )
    @mock.patch("betfairlightweight.baseclient.BaseClient.keep_alive_headers")
    @mock.patch("betfairlightweight.baseclient.requests.post")
    def test_request_json_error(
        self, mock_post, mock_keep_alive_headers, mock_json_loads
    ):
        mock_response = mock.Mock()
        mock_response.status_code = 200
        mock_post.return_value = mock_response

        with self.assertRaises(InvalidResponse):
            self.keep_alive.request()

    def test_keep_alive_error_handler(self):
        mock_response = create_mock_json("tests/resources/keep_alive_success.json")
        assert self.keep_alive._error_handler(mock_response.json()) is None

        mock_response = create_mock_json("tests/resources/keep_alive_fail.json")
        with self.assertRaises(KeepAliveError):
            self.keep_alive._error_handler(mock_response.json())

    def test_url(self):
        assert self.keep_alive.url == "https://identitysso.betfair.com/api/keepAlive"


import pandas as pd
import sys

def parse_file(filename):
	df = pd.read_csv(filename,sep = ',',header = 0,names = ["id","skill","numberInt","type","isActive"])
	Skills = set(df["skill"].astype("U").to_numpy())

	for skill in Skills:
		print(skill)

def main():
	FILENAME = sys.argv[1]
	parse_file(FILENAME)

if __name__ == '__main__':
	main()