import torch

def accuracy(prob, labels):
    '''
        Calculate the accuracy (= sum(no. of sample that prediction is fully correct)/batch_size)

        Args:
            prob: a batch of predictions in probability from model
            labels: a batch of labels from data loader
    '''
    prob = prob.detach().cpu().numpy()
    labels = labels.detach().cpu().numpy()
    preds = prob>0.5
    return ((labels==preds).sum(axis=1)==4).sum() / len(preds)
    # return (((labels==preds).sum(axis=1)==4).sum() / len(preds)).item()

def _take_channels(*xs, ignore_channels=None):
    if ignore_channels is None:
        return xs
    else:
        channels = [channel for channel in range(xs[0].shape[1]) if channel not in ignore_channels]
        xs = [torch.index_select(x, dim=1, index=torch.tensor(channels).to(x.device)) for x in xs]
        return xs

def _threshold(x, threshold=None):
    if threshold is not None:
        return (x > threshold).type(x.dtype)
    else:
        return x

def iou(pr, gt, eps=1e-7, threshold=0.5, ignore_channels=None):
    """Calculate Intersection over Union between ground truth and prediction
    Args:
        pr (torch.Tensor): predicted tensor
        gt (torch.Tensor):  ground truth tensor
        eps (float): epsilon to avoid zero division
        threshold: threshold for outputs binarization
    Returns:
        float: IoU (Jaccard) score
    """

    pr = _threshold(pr, threshold=threshold)
    pr, gt = _take_channels(pr, gt, ignore_channels=ignore_channels)

    intersection = torch.sum(gt * pr)
    union = torch.sum(gt) + torch.sum(pr) - intersection + eps
    return ((intersection + eps) / union).item()


import community as community_louvain # pip install python-louvain
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import networkx as nx

random_state = 2022

# helper function to load data in from file
def get_data(filename):
    # use a context manager to load in the data
    with open(filename, 'r', encoding='utf8') as rf:
        # transform file into string and split along new line
        lines = rf.read().split("\n")

        # separate each line along the tab characters
        data = [line.split("\t") for line in lines]

        # grab the header
        header = data[0]

        # delete header from data
        data = data[1:]

    # return header and data
    return header, data

# load data in from file
node_header, node_data = get_data('sw_network_nodes_allCharacters.csv')
edge_header, edge_data = get_data('sw_network_edges_allCharacters.csv')

# create graph object
G = nx.Graph()

episode = "episode-5"
episodeNodes = []

# add edge information to the graph
for edge in edge_data:
    # add edge one by one, node 1, node 2, weight
    if episode in edge[3]:
        episodeNodes.append(edge[0])
        episodeNodes.append(edge[1])
        G.add_edge(edge[0], edge[1], weight=int(edge[2]))

# add node information to the graph
for node in node_data:
    if node[0] in episodeNodes:
        # add nodes one by one, with id, label, affiliation, and side
        G.add_node(node[0], label=node[1], affiliation=node[2], side=node[3])

# metrics
degree_centrality = nx.degree_centrality(G)
closeness_centrality = nx.closeness_centrality(G)
betweenness_centrality = nx.betweenness_centrality(G)

# visualize this
#nx.draw_spring(G)
#plt.show()

#G = nx.karate_club_graph()

#first compute the best partition
partition = community_louvain.best_partition(G)

# draw the graph
pos = nx.spring_layout(G, seed=random_state) # layout is always a bit random, hence we use `seed`
# color the nodes according to their partition
cmap = cm.get_cmap('viridis', max(partition.values()) + 1)
nx.draw_networkx_nodes(G, pos, partition.keys(), node_size=40,
                       cmap=cmap, node_color=list(partition.values()))
nx.draw_networkx_edges(G, pos, alpha=0.5)
nx.draw_networkx_labels(G, pos, font_size=10)
plt.show()


"""
Log setting model
"""
import dataclasses
import json
import os
import logging
import logging.config

@dataclasses.dataclass
class OrgLog:
    """ Log setting model """

    def __init__(self, conf_file='conf/log_format.json', logger_name=None):
        if not os.path.isdir('../log'):
            os.makedirs('../log', exist_ok=True)
        self._read_conf_file(conf_file)
        self.logger = logging.getLogger(logger_name)

    @classmethod
    def _read_conf_file(cls, conf_file: str):
        """ Loading a format file """
        with open(conf_file, 'r', encoding='utf-8') as conf_f:
            conf_file = json.load(conf_f)
            logging.config.dictConfig(conf_file)

    def _test(self):
        """ Test Run """
        self._read_conf_file(conf_file='../conf/log_format.json')
        self.logger.info('test info')
        self.logger.error('test error')


import datasets
import numpy
import pandas
import torch
from transformers import DistilBertTokenizer

def fetch_data(data_split = "train"):
    dataset = datasets.load_dataset("health_fact")
    if data_split == "train":
        data = pandas.DataFrame(dataset["train"])
    elif data_split == "validation":
        data = pandas.DataFrame(dataset["validation"])
    else:
        data = pandas.DataFrame(dataset["test"])
    return data

def remove_bad_observations(data):
    good_indices = numpy.asarray(data["label"] != -1).nonzero()[0] #access tuple element
    cleaned_data = data.iloc[good_indices]
    cleaned_data.reset_index(inplace = True) #needed to avoid torch compatibility issues

    claims = cleaned_data["claim"].tolist()
    explanations = cleaned_data["explanation"].tolist()
    labels = torch.tensor(cleaned_data["label"])
    return claims, explanations, labels

def tokenize(tokenizer, claims, explanations):
    output = tokenizer(claims,
                       explanations,
                       padding = "max_length",
                       truncation = "longest_first",
                       return_tensors = "pt",
                       return_attention_mask = True)
    input_ids = output["input_ids"]
    attention_mask = output["attention_mask"]
    return input_ids, attention_mask

def preprocess(tokenizer, data_split = "train"):
    data = fetch_data(data_split)
    claims, explanations, labels = remove_bad_observations(data)
    input_ids, attention_mask = tokenize(tokenizer, claims, explanations)
    return input_ids, attention_mask, labels

if __name__ == "__main__":
    tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
    input_ids, attention_mask, labels = preprocess(tokenizer)


def Mi_Funcion (Nombre,Apellido):
    print(f'Nombre: {Nombre}, Apellido: {Apellido}')

Mi_Funcion('Pepe', 'Acosta')

def sumando(a,b):
    return a+b

vValor1 = input('Ingrese número entero a sumar: ')
while not vValor1.isdigit(): #Validamos que se informe un valor numérico
	print('Debe ingresar un valor numérico, intente de nuevo')
	vValor1 = input('Ingrese número entero a sumar: ')

vValor2 = input('Ingrese el segundo valor entero a sumar: ')
while not vValor2.isdigit():#Validamos que se informe un valor numérico
	print('Debe ingresar un valor numérico, intente de nuevo')
	vVar2 = input('Ingrese un número entero a sumar: ')

vValor1 = int(vValor1) #Convertimos a tipo entero
vValor2 = int(vValor2) #Convertimos a tipo entero

resultado =sumando(vValor1,vValor2) #Ejecutamos la función, pasamos los argumentos
print(f'El resultado de la suma es: {resultado}')


#https://open.kattis.com/problems/pot

from sys import stdin

total = 0
numbers = stdin.readline()

for num in stdin:
    num = num.strip()
    total += int(num[:-1]) ** int(num[-1])

print(total)


import numpy as np
from dragons import meraxes
import os
import matplotlib.pyplot as plt
import sys

#Setup
cosmo = {'omega_M_0' : 0.308,
'omega_lambda_0' : 0.692, 'omega_b_0' : 0.04839912,
'omega_b_0' : 0.04839912,
'omega_n_0' : 0.0,
'N_nu' : 0,
'h' : 0.678,
'n' : 0.968,
'sigma_8' : 0.815
}

snaps=[52,63,78,100,116,134,158];
n_red=len(snaps)
n_split=5
f1_bin=np.zeros((n_split+1,n_red+1));
f2_bin=np.zeros((n_split+1,n_red+1));
f3_bin=np.zeros((n_split+1,n_red+1));
f4_bin=np.zeros((n_split+1,n_red+1));
f5_bin=np.zeros((n_split+1,n_red+1));
M_bin=np.zeros((n_split+1,n_red+1));
G=43.01

#Import simulation data
data_folder='/home/mmarshal/data_dragons/'
meraxes_loc='/output/meraxes.hdf5'
filename='bulges_update1014_proj'

ss=0
color=['k','b','r','g','m','purple','orange']
fig, axes = plt.subplots(1,1)
for snapshot in snaps:
  gals=meraxes.io.read_gals(data_folder+filename+meraxes_loc,\
  snapshot=snapshot,\
  h=cosmo['h'],quiet=True)
  gals=gals[gals['StellarMass']>0]

  f1=G*(gals['StellarMass']-gals['BulgeStellarMass'])*gals['ColdGas']*gals['StellarDiskScaleLength']/(gals['StellarDiskScaleLength']+gals['GasDiskScaleLength'])**2
  f2=G*(gals['StellarMass']-gals['BulgeStellarMass'])**2/(4*gals['StellarDiskScaleLength'])
  f3=(gals['StellarMass']-gals['BulgeStellarMass'])*(gals['Vvir']**2)
  f4=(gals['StellarMass']-gals['BulgeStellarMass'])*(gals['VStellarDisk']**2)/2
#  f5=G*(gals['StellarMass']-gals['BulgeStellarMass'])*gals['BulgeStellarMass']/gals['StellarDiskScaleLength']
  sigmav=10**(0.31*np.log10(gals['BulgeStellarMass']*1e10)-1.15)
  print(np.mean(sigmav))
  f5=(gals['StellarMass']-gals['BulgeStellarMass'])*((np.sqrt(3)*sigmav)**2)
  Mvir=np.log10(gals['Mvir'])+10
  M_split=(max(Mvir)-min(Mvir))/n_split
  for ii in range(0,n_split):
    f1_bin[ii,ss]=np.nanmedian(f1[(Mvir>min(Mvir)+M_split*ii)&(Mvir<min(Mvir)+M_split*(ii+1))])
    f2_bin[ii,ss]=np.nanmedian(f2[(Mvir>min(Mvir)+M_split*ii)&(Mvir<min(Mvir)+M_split*(ii+1))])
    f3_bin[ii,ss]=np.nanmedian(f3[(Mvir>min(Mvir)+M_split*ii)&(Mvir<min(Mvir)+M_split*(ii+1))])
    f4_bin[ii,ss]=np.nanmedian(f4[(Mvir>min(Mvir)+M_split*ii)&(Mvir<min(Mvir)+M_split*(ii+1))])
    f5_bin[ii,ss]=np.nanmedian(f5[(Mvir>min(Mvir)+M_split*ii)&(Mvir<min(Mvir)+M_split*(ii+1))])
    M_bin[ii,ss]=(min(Mvir)+M_split*ii+min(Mvir)+M_split*(ii+1))/2
  axes.plot(M_bin[:,ss],np.log10(f1_bin[:,ss]/f4_bin[:,ss]),':',color=color[ss])
  axes.plot(M_bin[:,ss],np.log10(f2_bin[:,ss]/f4_bin[:,ss]),'-.',color=color[ss])
  axes.plot(M_bin[:,ss],np.log10(f3_bin[:,ss]/f4_bin[:,ss]),'-',color=color[ss])
  axes.plot(M_bin[:,ss],np.log10(f5_bin[:,ss]/f4_bin[:,ss]),'--',color=color[ss])
  ss+=1

lines=axes.get_lines()
legend1=plt.legend([lines[i] for i in [0,1,2,3]],['PE Gas Disk / PE Halo','PE Stellar Disk / PE Halo','KE / PE Halo','PE Bulge / PE Halo'],loc=2)
axes.add_artist(legend1)
plt.legend([lines[i] for i in [0,4,8,12,16,20,24]],['z=8','z=7','z=6','z=5','z=4','z=3','z=2'],loc=4)
#axes.add_artist(legend2)
plt.xlabel('log(Halo Mass)')
plt.ylabel('log(Fraction)')
plt.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author:  kerwin.cn@gmail.com
# Created Time:2017-09-20 20:49:18
# Last Change:  2017-12-17 23:06:04
# File Name: sample1.py
import backtrader as bt
import CerebroBase
import StrategyBase

# Create a Stratey
# 这个MACD系统
class Strategy_MACD(StrategyBase.StrategyBase):
    """

    """

    params = (('fastperiod', 12),
              ('slowperiod', 26),
              ('signalperiod', 9),
              ('ma1', 10),
              )

    def __init__(self):
        # 这个就是那个柱状
        self.macdhisto = bt.indicators.MACDHisto(self.data)
        self.sma1 = bt.talib.SMA(self.data, timeperiod=self.params.ma1)
        self.order = None

    def next(self):
        # Simply log the closing price of the series from the reference
        # self.log('Close, %.2f' % self.dataclose[0])

        # Check if an order is pending ... if yes, we cannot send a 2nd one
        if self.order:
            return

        size1 = self.get_right_sizing()
        _close = self.data_close[0]
        _sma1 = self.sma1[0]

        # if self.position.size > 0:
            # # 有买单，判断止损是小于均线
            # if _close < _stop_price:
                # self.order = self.close()
            # pass
        # elif self.position.size < 0:
            # # 有卖单，判断止损是大于均线
            # if _close > _stop_price:
                # self.order = self.close()
            # pass
        # 这里判断是否下单
        if(self.macdhisto[0] > 0
           and self.macdhisto[-1] < 0
           and _close > _sma1):
            self.order = self.buy(size=size1)
        if(self.macdhisto[0] < 0
           and self.macdhisto[-1] > 0
           and _close < _sma1):
            self.order = self.sell(size=size1)

if __name__ == '__main__':
    # Create a cerebro entity
    cerebro = CerebroBase.CerebroAGTD()
    # Add a strategy
    cerebro.addstrategy(Strategy_MACD)
    # Set our desired cash start
    cerebro.set_cash(5000.0)
    cerebro.show_analyzer(cerebro.run())
    cerebro.show_plot()


"""
run with
python setup.py install; nosetests --nocapture -v tests/test_jobdb.py
python setup.py install; nosetests tests/test_jobdb.py
python setup.py install; nosetests -v --nocapture tests/test_jobdb.py:TestJobDB.test_001_start

"""
from __future__ import print_function

from cloudmesh_job.cm_jobdb import JobDB
from cloudmesh_base.util import HEADING
from cloudmesh_base.Shell import Shell
import os

# nosetest --nocapture
class TestJobDB:

    def setup(self):
        # HEADING()
        self.db = JobDB()

    def teardown(self):
        # HEADING()
        pass

    def test_001_start(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        self.db.start()
        up = self.db.isup()
        result = up
        assert result

    def test_002_up(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        up = self.db.isup()
        print (up)
        self.db.stop()
        down = not self.db.isup()
        self.db.start()
        assert up and down

    def test_003_pid(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        pid = self.db.pid()
        print (pid)
        assert True

    def test_004_connect(self):
        """
        tests if a mongo db can be connected to
        :return:
        """
        HEADING()
        self.db.connect()

        result = True
        assert result

    def test_005_clear(self):
        """
        tests clearing all jobs from the db
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        db.clear()

        # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
        assert(len(db) == 0)

    def test_006_add(self):
        """
        tests adding jobs to the db
        :return:
        """
        HEADING()
        db = self.db

        count = 5

        db.connect()

        db.delete_jobs()

        for id in range(0,count):
            job = db.insert("job" + str(id))

        assert len(db) == count

    def test_007_delete(self):
        """
        tests deleting a single job from the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        print ("AAA")
        before_count = len(db)
        print ("CCC", len(db))
        job = db.insert("deleteme")
        print ("DDD", len(db))

        job = db.delete_jobs("job_name", "deleteme")
        print ("EEE")
        after_count = len(db)
        print ("FFF", len(db))
        assert(before_count - after_count == 0)

    def test_008_modify(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        job = {"job_name": "modifyme", "input_filename":"file1"}

        db.add(job)

        originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]

        job = {"job_name": "modifyme", "input_filename":"file2"}

        db.modify(job)

        newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]

        assert(originalFilename != newFilename)

    def test_010_info(self):
        """
        prints the info about the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        db.info()
        pass

    def test_011_len(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        count = len(db)
        print (count)
        assert count == 6

    def test_012_yaml_load(self):
        """
        tests adding jobs from a YAML file
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")

        count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))

        # Assert that the correct number jobs have been added
        assert(db.count() == count_fgrep)

    def test_013_find_files(self):
        """
        tests searching for a file
            the file being searched for exists in 3 files: twice as input and twice as output
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")
        inputs, outputs = db.find_jobs_with_file("in1.txt")

        # Assert that the lengths of the inputs and outputs arrays are correct
        count_fgrep = len(Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
        assert(len(inputs) == count_fgrep)

    def test_14_jobid(self):
        HEADING()
        db = self.db

        db.connect()

        db.set_jobid(11)

        a = db.get_jobid()
        db.incr_jobid()
        b = db.get_jobid()

        print (a, b)
        assert b == a + 1

    def test_15_jobscript(self):
        HEADING()
        db = self.db
        db.connect()

        name = "test1"
        contents = "hallo"

        db.add_script(name, contents)
        db.add_script("test2", "other")
        script = db.get_script(name)

        print ("Script:", script)
        print ("Content:", contents)

        db.write_script(name, "/tmp/script.txt")
        what = Shell.cat("/tmp/script.txt")

        assert contents == what
        assert contents == script

    def test_999_stop(self):
        """
        tests if the mongo db can be shutdown
        :return:
        """
        HEADING()
        self.db.stop()
        result = True
        assert result


import glob
import os
from shutil import copyfile
from pprint import pprint
import pickle
from PIL import Image
import numpy as np
from autoPyTorch import AutoNetClassification, HyperparameterSearchSpaceUpdates
import sklearn.model_selection
import sklearn.metrics
from sklearn import preprocessing

input_size = (256,256)

def get_data_references(images = [] , labels = [] , filter = "_positive.txt", copy_to = "/tmp/covid_dataset", balance = 180): #TODO: add balanced/imbalanced and number of examples limit
    '''
    If you don't  want to copy set copy_to = None
    balance = 180 means each class gets 180 examples before splits
    '''
    for name in glob.glob("../data/*{}".format(filter)):
        for i, imagepaths in enumerate(open(name).readlines()):
            if i==balance:
                break
            images.append(imagepaths)
            labels.append(name.split(filter)[0])
    le = preprocessing.LabelEncoder()
    le.fit(list(set(labels)))
    #Create sklearn type labels
    labels = le.transform(labels)
    print("Found {} examples with labels {}".format(len(labels), le.classes_))
    #Copy data locally if needed
    if copy_to:
        os.makedirs(copy_to, exist_ok=True)
        for i, im in enumerate(images):
            if "\n" in im:
                im = im.strip()
            dest = os.path.join(copy_to, "{0}.{1}".format(str(i), im.split("/")[-1].split(".")[-1]))
            copyfile(im, dest)
            images[i] = dest
        print("Copy {} files".format(i+1))
    assert len(images) == len(labels)
    return images, labels, le

def jpg_image_to_array(image_path):
    '''
    Loads JPEG imag
    '''
    with Image.open(image_path).convert("RGB") as image:
        image = image.resize(input_size)
        im_arr = np.fromstring(image.tobytes(), dtype=np.uint8) / 255.0
        try:
            im_arr = im_arr.reshape((image.size[1], image.size[0], 3))
        except ValueError as e:
            im_arr = im_arr.reshape((image.size[1], image.size[0]))
            im_arr = np.stack((im_arr,)*3, axis=-1)
    return im_arr

if __name__ == "__main__":
    #Get the dataset (this could be yielded/batched)
    images, labels, le  = get_data_references()
    #Read the images
    X = []
    for i, im in enumerate(images):
        image = jpg_image_to_array(im)
        X.append(image)
    #autopytorch format
    X = [np.asarray(x).flatten() for x in X]
    #Split the dataset
    X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(np.asarray(X, dtype=np.float16), np.asarray(labels, dtype=np.uint8), random_state=1, test_size=0.1, shuffle=True)
    scaler = preprocessing.StandardScaler()
    scaler.fit(X_train)
    X_train = scaler.transform(X_train)
    X_test = scaler.transform(X_test)
    #Init custom search space
    # search_space_updates = HyperparameterSearchSpaceUpdates()
    # search_space_updates.append(node_name="CreateDataLoader",
    #                             hyperparameter="batch_size",
    #                             value_range=[32],
    #                             log=False)
    #Init autonet
    # autoPyTorch = AutoNetClassification(hyperparameter_search_space_updates=search_space_updates,  # config
    autoPyTorch = AutoNetClassification(
                                        # "full_cs",  # config
                                        networks=["resnet"],
                                        # torch_num_threads=2,
                                        log_level='info',
                                        budget_type='epochs',
                                        min_budget=5,
                                        max_budget=20,
                                        num_iterations=100,
                                        cuda=True, use_pynisher=False)
    #fit
    autoPyTorch.fit(X_train=X_train, Y_train=y_train, X_valid=X_test, Y_valid=y_test, optimize_metric="auc_metric", loss_modules=["cross_entropy", "cross_entropy_weighted"])
    #predict
    y_pred = autoPyTorch.predict(X_test)
    #check
    print("Accuracy score", sklearn.metrics.accuracy_score(y_test, y_pred))
    print("Confusion matrix", sklearn.metrics.confusion_matrix(y_test, y_pred, labels=le.classes_))
    print(autoPyTorch)
    pytorch_model = autoPyTorch.get_pytorch_model()
    print(pytorch_model)


import numpy as np

def deposit_particles(xyz, weights=None, bins=(32,32,32), range=None):
    """
    Simple histogramming of particles.

    Parameters
    ----------
    xyz : np.array
        Array of particle positions in [m[
        with shape (n_particles, 3)

    weights: np.array
        Array of weights in [C]

    Returns
    -------
    tupule of

    rho_mesh : np.array
            Charge mesh
            rho_mesh.sum() will equal weights.sum(), the total charge in [C]

    deltas : (float, float, float)
        Mesh spacing in [m]

    coord_vecs : [np.array, np.array, np.array]
        coordinate vectors to create coordinate meshes, e.g.
            X, Y, Z = np.meshgrid(*coord_vecs, indexing='ij')  # Must have this indexing!!!

    """
    rho_mesh, edges = np.histogramdd(xyz, bins=bins, range = range, weights=weights )

    # Get deltas
    deltas = np.array([np.mean(np.diff(e)) for e in edges])
    mins = [e.min()+deltas[i]/2 for i, e in enumerate(edges)]

    coord_vecs = [np.arange(0,n,1)*d+m for n, d, m in zip(rho_mesh.shape, deltas, mins)]

    return rho_mesh, deltas, coord_vecs


from __future__ import annotations

import json
import typing as t

import fsspec
import numpy as np

from squirrel.serialization.serializer import SquirrelSerializer

if t.TYPE_CHECKING:
    from fsspec.spec import AbstractFileSystem

    from squirrel.constants import ShardType

class SquirrelJsonEncoder(json.JSONEncoder):
    """Json encoder for numpy types."""

    def default(self, obj: t.Any) -> t.Any:
        """The default function to encode numpy types."""
        if isinstance(
            obj,
            (
                np.int_,
                np.intc,
                np.intp,
                np.int8,
                np.int16,
                np.int32,
                np.int64,
                np.uint8,
                np.uint16,
                np.uint32,
                np.uint64,
            ),
        ):
            return int(obj)
        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
            return float(obj)
        elif isinstance(obj, (np.ndarray,)):
            return {"data": obj.tolist(), "type": obj.dtype.str, "shape": obj.shape}
        return json.JSONEncoder.default(self, obj)

class SquirrelJsonDecoder(json.JSONDecoder):
    """Json decoder for numpy types."""

    def __init__(self, *args, **kwargs):
        """Initialize SquirrelJsonDecoder."""
        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)

    def object_hook(self, dct: t.Dict) -> t.Any:
        """Decode custom types."""
        if "type" in dct and "data" in dct and "shape" in dct:
            return np.array(dct["data"], dtype=np.dtype(dct["type"])).reshape(dct["shape"])
        return dct

class JsonSerializer(SquirrelSerializer):
    def __init__(self, deser_hook: t.Optional[t.Callable] = None) -> None:
        """Initializes JsonSerializer.

        Args:
            deser_hook (Callable): Callable that is passed as `object_hook` to :py:class:`JsonDecoder` during json
                deserialization. Defaults to None.
        """
        super().__init__()
        self.deser_hook = deser_hook

    @staticmethod
    def serialize(obj: t.Any) -> bytes:
        """Returns the object serialized with json."""
        return json.dumps(obj, cls=SquirrelJsonEncoder).encode("utf-8")

    def deserialize(self, obj: bytes) -> t.Any:
        """Returns the object deserialized with json."""
        return json.loads(obj, cls=SquirrelJsonDecoder, object_hook=self.deser_hook)

    @staticmethod
    def serialize_shard_to_file(
        shard: ShardType,
        fp: str,
        fs: t.Optional[AbstractFileSystem] = None,
        mode: str = "wb",
        **open_kwargs,
    ) -> None:
        """Writes a shard to a file by only writing and serializing the values of its samples.

        Args:
            shard (ShardType): Shard to serialize and write to the file.
            fp (str): Path to the file to write.
            fs (AbstractFileSystem, optional): Filesystem to use for opening the file. If not provided, `fsspec` will
                pick a filesystem suitable for `fp`. Defaults to None.
            mode (str): IO mode to use. Passed to :py:meth:`fs.open`. Defaults to "wb".
            **open_kwargs: Other keyword arguments passed to :py:meth:`fs.open`. `open_kwargs` will always have
                `compression="gzip"` set.
        """
        open_kwargs["mode"] = mode
        open_kwargs["compression"] = "gzip"

        if fs is None:
            fs = fsspec

        with fs.open(fp, **open_kwargs) as f:
            for sample in shard:
                json_string = json.dumps(sample, cls=SquirrelJsonEncoder) + "\n"
                f.write(json_string.encode("utf-8"))

    @staticmethod
    def deserialize_shard_from_file(
        fp: str, fs: t.Optional[AbstractFileSystem] = None, mode: str = "rb", **open_kwargs
    ) -> t.Iterable[t.Any]:
        """Reads a shard from file and returns an iterable over its samples.

        Args:
            fp (str): Path to the file to write.
            fs (AbstractFileSystem, optional): Filesystem to use for opening the file. If not provided, `fsspec` will
                pick a filesystem suitable for `fp`. Defaults to None.
            mode (str): IO mode to use. Passed to :py:meth:`fs.open`. Defaults to "rb".
            **open_kwargs: Other keyword arguments passed to :py:meth:`fs.open`. `open_kwargs` will always have
                `compression="gzip"` set.

        Yields:
            (Any) Values of the samples of the shard.
        """
        open_kwargs["mode"] = mode
        open_kwargs["compression"] = "gzip"

        if fs is None:
            fs = fsspec

        with fs.open(fp, **open_kwargs) as f:
            for sample_bytes in f:
                yield json.loads(sample_bytes, cls=SquirrelJsonDecoder)


import argparse
import numpy as np
import sys
import xarray

from splitvar import *

def parse_args(args):

    parser = argparse.ArgumentParser(description='Split multiple netCDF files by time and variable')

    parser.add_argument('--verbose',
                        help='Verbose output',
                        action='store_true')
    parser.add_argument('-f','--frequency',
                        help='Time period to group for output',
                        default='Y',
                        action='store')
    parser.add_argument('--aggregate',
                        help='Apply mean in time, using pandas frequency notation e.g Y, 6M, 2Y',
                        default=None,
                        action='store')
    # parser.add_argument('--function',
    #                     help='Function to apply to aggregation',
    #                     default='mean',
    #                     action='store')
    parser.add_argument('-v','--variables',
                        help='Only extract specified variables',
                        action='append')
    parser.add_argument('-x','--x-variables',
                        default=[],
                        dest='delvars',
                        help='Delete specified variables from input data',
                        action='append')
    parser.add_argument('-d','--delattr',
                        help='Delete specified global attributes',
                        default=['filename'],
                        action='append')
    parser.add_argument('-a','--add',
                        help='Read in additional variables from these files',
                        default=[],
                        action='append')
    parser.add_argument('-s','--skipvars',
                        help='Do not extract these variables',
                        default=['time'],
                        action='append')
    parser.add_argument('-t','--title',
                        help='Title of the simulation, included in metadata')
    parser.add_argument('--simname',
                        help='Simulation name to include in the filename')
    parser.add_argument('--model-type',
                        dest='modeltype',
                        help='Model type to include in the filename',
                        default='')
    parser.add_argument('--timeformat',
                        help='strftime format string for date fields in filename',
                        default='%Y%m')
    parser.add_argument('--timeshift',
                        help='Shift time axis by specified amount (in whatever units are used in the file). Default is to automatically shift current start date to time origin',
                        const='auto',
                        nargs='?')
    parser.add_argument('--usebounds',
                        help='Use mid point of time bounds for time axis',
                        action='store_true')
    parser.add_argument('--datefrombounds',
                        help='Use time bounds for filename datestamp',
                        action='store_true')
    parser.add_argument('--calendar',
                        help='Specify calendar: will replace value of calendar attribute whereever it is found')
    parser.add_argument('-o','--outputdir',
                        help='Output directory in which to store the data',
                        default='.')
    parser.add_argument('--overwrite',
                        help='Overwrite output file if it already exists',
                        action='store_true')
    parser.add_argument('-cp','--copytimeunits',
                        help='Copy time units from time variable to bounds',
                        action='store_true')
    parser.add_argument('--makecoords',
                        help='Create variable for any dimension without corresponding coordinate variable as per CF convention',
                        action='store_true')
    parser.add_argument('--engine',
                        help='Back-end used to write output files (options are netcdf4 and h5netcdf)',
                        default='netcdf4')
    parser.add_argument('--deflate',
                        help='Deflate compression level',
                        default=5,
                        choices=range(0, 10))
    parser.add_argument('--filecachesize',
                        help='Number of files xarray keeps in cache. For large datasets this may need to be set to a lower value to avoid excessive memory use (default=128)',
                        type=int)
    parser.add_argument('inputs', help='netCDF files', nargs='+')

    return parser.parse_args(args)

def main_parse_args(args):
    '''
    Call main with list of arguments. Callable from tests
    '''
    # Must return so that check command return value is passed back to calling routine
    # otherwise py.test will fail
    return main(parse_args(args))

def main_argv():
    '''
    Call main and pass command line arguments. This is required for setup.py entry_points
    '''
    main_parse_args(sys.argv[1:])

def main(args):

    verbose = args.verbose

    if args.filecachesize:
        xarray.set_options(file_cache_maxsize=args.filecachesize)

    # Open first file in series to determine dependencies and variables
    # needed to load the full dataset. Don't specify delvars on open,
    # delete after
    ds = open_files(args.inputs[0], None, args.delvars)

    # Find the time coordinate. Will return the first one. Code doesn't
    # support multiple time axes
    try:
        timevar = findmatchingvars(ds, matchstrings=[' since '], coords_only=True)[0]
    except IndexError:
        print('No time coordinate found! Aborting')
        raise
    if verbose: print('Found time coordinate: {}'.format(timevar))

    # Add additional variables, such as grid information. Need to add
    # at this stage to properly determing dependencies
    ds = add_vars(ds, args.add, timevar)

    # Need this step to make sure the dependencies are correct later
    if args.makecoords:
        # Loop over all dimensions without coordinates and make a
        # variable that is the dimension indexed by itself
        for var in set(ds.dims.keys()).difference(set(ds.coords.keys())):
            ds[var] = xr.DataArray(ds[var].values.astype(dtype=np.float32),
                                   coords={var:ds[var].values.astype(dtype=np.float32)},
                                   dims=[var])

    # Create a dictionary we can use to find dependent vars
    # for a given variable
    depvars = getdependents(ds)

    # Mapping from dependent variables back to variables which
    # depend on them
    is_dependent = dependentlookup(depvars)

    # Grab the list of variables required for output default to all
    # data variables in the dataset if none specified
    variables = args.variables
    if variables is None:
        variables = set(ds.variables)
    else:
        variables = set(variables).intersection(set(ds.variables))
        # Add back in dependent variables. Loop over list(variables) as
        # variables is being modified
        for var in list(variables):
            variables.update(depvars[var])

    # Remove variables specified to be deleted. Useful for removing
    # variables which can then be updated globally, e.g. grid variables
    # that change during a run, but should be defined to be one value
    variables.difference_update(args.delvars)

    # Check encoding options
    if args.deflate > 0:
        encoding = { 'zlib': True, 'shuffle': True, 'complevel': args.deflate }
    else:
        encoding = {}

    # Open full dataset and exclude all variables that aren't
    # in vars
    ds = open_files(args.inputs, timevar, set(ds.variables).difference(variables), verbose, encoding)

    # Add auxiliary data
    ds = add_vars(ds, args.add, timevar)

    if verbose:
        print('Opened source data:\n')
        print(ds)

    if args.simname:
        ds.attrs['simname'] = args.simname
    if 'simname' not in ds.attrs:
        ds.attrs['simname'] = 'simname'
    ds.attrs['simname'] = sanitise(ds.attrs['simname'])

    if args.title:
        ds.attrs['title'] = args.title
    if 'title' in ds.attrs:
        ds.attrs['title'] = sanitise(ds.attrs['title'])

    for attr in ['calendar', 'calendar_type']:
        if args.calendar:
            ds[timevar].attrs[attr] = args.calendar
        try:
            ds[timevar].attrs[attr] = ds[timevar].attrs[attr].lower()
        except:
            pass

    # In some cases the units in the time bounds is just 'days'
    # which leads to them being decoded as timedelta. Setting
    # this command line option will copy the units from time
    # into the bounds variable
    boundsvar = None
    if args.copytimeunits:
        if 'bounds' in ds[timevar].attrs:
            boundsvar = ds[timevar].attrs['bounds']
            ds[boundsvar].attrs['units'] = ds[timevar].attrs['units']

    if args.timeshift:
        # Apply a timeshift to all variables with a time axis
        if args.timeshift == 'auto':
            if boundsvar is not None:
                shift = ds[boundsvar].values[0][0]
            else:
                shift = ds[timevar].values[0]
        else:
            shift = float(args.timeshift)
        for var in findmatchingvars(ds, matchstrings=[' since ']):
            ds[var] = ds[var].copy(data = (ds[var].values[:] - shift))

    if args.usebounds:
        # Replace time variable with the average of the time bounds. Useful
        # for time variables that are defined at the end of a month, rather
        # than the middle
        if 'bounds' in ds[timevar].attrs:
            boundsvar = ds[timevar].attrs['bounds']
            newtime = [(e.values - b.values)//2 + b.values for (b,e) in ds[boundsvar]]
            ds[timevar] = ds[timevar].copy(data = newtime)

    if args.makecoords:
        # Loop over all dimensions without coordinates and make a
        # variable that is the dimension indexed by itself
        for var in set(ds.dims.keys()).difference(set(ds.coords.keys())):
            ds[var] = xr.DataArray(ds[var].values.astype(dtype=np.float32),
                                   coords={var:ds[var].values.astype(dtype=np.float32)},
                                   dims=[var])

    ds = xarray.decode_cf(ds)

    # Add all dependent variables to the skipvar list
    skipvars = set(args.skipvars + list(is_dependent.keys()))

    for var in splitbyvar(ds, args.variables, skipvars, verbose):
        i = 0
        print('Splitting {var} by time'.format(var=var))
        name = sanitise(var)
        outpath = os.path.normpath(os.path.join(args.outputdir, ds.simname, args.modeltype, name))
        try:
            os.makedirs(outpath)
        except FileExistsError:
            pass
        varlist = [var,] + depvars[var]
        dsbyvar = ds[varlist]
        # Drop any variables xarray has automatically added that are not
        # superfluous. Especially important to not get spurious/confusing
        # coordinates
        try:
            dsbyvar = dsbyvar.drop_vars(set(dsbyvar.variables).difference(varlist))
        except AttributeError:
            dsbyvar = dsbyvar.drop(set(dsbyvar.variables).difference(varlist))

        if args.aggregate:
            dsbyvar = resamplebytime(dsbyvar, var, args.aggregate, timedim=timevar)
        for dsbytime in groupbytime(dsbyvar, freq=args.frequency, timedim=timevar):
            i += 1
            startdate = format_date(dsbytime[timevar].values[0], args.timeformat)
            enddate = format_date(dsbytime[timevar].values[-1], args.timeformat)
            if 'bounds' in dsbytime[timevar].attrs and args.datefrombounds:
                boundsvar = ds[timevar].attrs['bounds']
                startdate = format_date(dsbytime[boundsvar].values[0][0], args.timeformat)
                enddate = format_date(dsbytime[boundsvar].values[-1][1], args.timeformat)
            fname = '{name}_{simulation}_{fromdate}_{todate}.nc'.format(
                        name=name,
                        simulation=ds.attrs['simname'],
                        fromdate=startdate,
                        todate=enddate,
                     )
            fpath = os.path.join(outpath, fname)
            if os.path.exists(fpath) and not args.overwrite:
                print("Output file {} already exists, and --overwrite not enabled. Skipping".format(fpath))
                continue

            dsbytime.attrs['time_coverage_start'] = startdate
            dsbytime.attrs['time_coverage_end'] = enddate

            dsbytime.attrs['geospatial_lat_min'] =  99999.
            dsbytime.attrs['geospatial_lat_max'] = -99999.
            for var in findmatchingvars(dsbytime, matchstrings=['degrees_N', 'degrees_north']):
                if var in dsbytime:
                    dsbytime.attrs['geospatial_lat_min'] = min(
                        dsbytime[var].min().values, dsbytime.attrs['geospatial_lat_min'])
                    dsbytime.attrs['geospatial_lat_max'] = max(
                        dsbytime[var].max().values, dsbytime.attrs['geospatial_lat_max'])

            dsbytime.attrs['geospatial_lon_min'] = 99999.
            dsbytime.attrs['geospatial_lon_max'] = -99999.
            for var in findmatchingvars(dsbytime, matchstrings=['degrees_E','degrees_east']):
                if var in dsbytime:
                    dsbytime.attrs['geospatial_lon_min'] = min(
                        dsbytime[var].min().values, dsbytime.attrs['geospatial_lon_min'])
                    dsbytime.attrs['geospatial_lon_max'] = max(
                        dsbytime[var].max().values, dsbytime.attrs['geospatial_lon_max'])

            for attr in list(dsbytime.attrs):
                try:
                    if abs(float(dsbytime.attrs[attr])) == 99999. :
                        del(dsbytime.attrs[attr])
                except:
                    pass

            for attr in args.delattr:
                try:
                    del(dsbytime.attrs[attr])
                except KeyError:
                    pass

            print(dsbytime)

            writevar(dsbytime, fpath, unlimited=timevar, engine=args.engine)
        if i == 0:
            # No data written, delete empty output directory
            os.rmdir(outpath)

if __name__ == '__main__':

    main_argv()


"""
Humanities Data Analysis: Case studies with Python
--------------------------------------------------
Folgert Karsdorp, Mike Kestemont & Allen Riddell
Chapter 2: Parsing and Manipulating Structured Data
"""

# %%
import tarfile
tf = tarfile.open('data/folger.tar.gz', 'r')
tf.extractall('data')

# %%
file_path = 'data/folger/txt/1H4.txt'
stream = open(file_path)
contents = stream.read()
stream.close()

print(contents[:300])

# %%
with open(file_path) as stream:
    contents = stream.read()

print(contents[:300])

# %%
with open('data/anna-karenina.txt', encoding='koi8-r') as stream:
    # Use stream.readline() to retrieve the next line from a file,
    # in this case the 1st one:
    line = stream.readline()

print(line)

# %%
csv_file = 'data/folger_shakespeare_collection.csv'
with open(csv_file) as stream:
    # call stream.readlines() to read all lines in the CSV file as a list.
    lines = stream.readlines()

print(lines[:3])

# %%
entries = []
for line in open(csv_file):
    entries.append(line.strip().split(','))

for entry in entries[:3]:
    print(entry)

# %%
import csv

entries = []
with open(csv_file) as stream:
    reader = csv.reader(stream, delimiter=',')
    for fname, author, title, editor, publisher, pubplace, date in reader:
        entries.append((fname, title))

for entry in entries[:5]:
    print(entry)

# %%
entries = []
with open(csv_file) as stream:
    reader = csv.reader(stream, delimiter=',')
    for fname, _, title, *_ in reader:
        entries.append((fname, title))

for entry in entries[:5]:
    print(entry)

# %%
a, _, c, _, _ = range(5)
print(a, c)

# %%
a, *l = range(5)
print(a, l)

# %%
seq = range(5)
a, l = seq[0], seq[1:]
print(a, l)

# %%
a, *l, b = range(5)
print(a, l, b)

# %%
entries = []

with open(csv_file) as stream:
    reader = csv.DictReader(stream, delimiter=',')
    for row in reader:
        entries.append(row)

for entry in entries[:5]:
    print(entry['fname'], entry['title'])

# %%
import PyPDF2 as PDF

# %%
file_path = 'data/folger/pdf/1H4.pdf'
pdf = PDF.PdfFileReader(file_path, overwriteWarnings=False)

# %%
n_pages = pdf.getNumPages()
print(f'PDF has {n_pages} pages.')

# %%
page = pdf.getPage(1)
content = page.extractText()
print(content[:150])

# %%
def pdf2txt(fname, page_numbers=None, concatenate=False):
    """Convert text from a PDF file into a string or list of strings.

    Arguments:
        fname: a string pointing to the filename of the PDF file
        page_numbers: an integer or sequence of integers pointing to the
            pages to extract. If None (default), all pages are extracted.
        concatenate: a boolean indicating whether to concatenate the
            extracted pages into a single string. When False, a list of
            strings is returned.

    Returns:
        A string or list of strings representing the text extracted
        from the supplied PDF file.

    """
    pdf = PDF.PdfFileReader(fname, overwriteWarnings=False)
    if page_numbers is None:
        page_numbers = range(pdf.getNumPages())
    elif isinstance(page_numbers, int):
        page_numbers = [page_numbers]
    texts = [pdf.getPage(n).extractText() for n in page_numbers]
    return '\n'.join(texts) if concatenate else texts

# %%
text = pdf2txt(file_path, concatenate=True)
sample = pdf2txt(file_path, page_numbers=[1, 4, 9])

# %%
import json

line = {
    'line_id': 12664,
    'play_name': 'Alls well that ends well',
    'speech_number': 1,
    'line_number': '1.1.1',
    'speaker': 'COUNTESS',
    'text_entry': 'In delivering my son from me, I bury a second husband.'
}

print(json.dumps(line))

# %%
with open('shakespeare.json', 'w') as f:
    json.dump(line, f)

# %%
with open('data/macbeth.json') as f:
    data = json.load(f)

print(data[3:5])

# %%
import collections

languages = collections.Counter()
for entry in data:
    languages[entry['lang']] += 1

print(languages.most_common())

# %%
with open('data/sonnets/18.xml') as stream:
    xml = stream.read()

print(xml)

# %%
import lxml.etree

# %%
tree = lxml.etree.parse('data/sonnets/18.xml')
print(tree)

# %%
# decoding is needed to transform the bytes object into an actual string
print(lxml.etree.tostring(tree).decode())

# %%
for rhyme in tree.iterfind('//rhyme'):
    print(f'element: {rhyme.tag} -> {rhyme.text}')

# %%
root = tree.getroot()
print(root.tag)

# %%
print(root.attrib['year'])

# %%
print(len(root))

# %%
children = [child.tag for child in root]

# %%
print('\n'.join(child.text or '' for child in root))

# %%
print(''.join(root[0].itertext()))

# %%
for node in root:
    if node.tag == 'line':
        print(f"line {node.attrib['n']: >2}: {''.join(node.itertext())}")

# %%
with open('data/sonnets/116.txt') as stream:
    text = stream.read()

print(text)

# %%
root = lxml.etree.Element('sonnet')
root.attrib['author'] = 'William Shakespeare'
root.attrib['year'] = '1609'

# %%
tree = lxml.etree.ElementTree(root)
stringified = lxml.etree.tostring(tree)
print(stringified)

# %%
print(type(stringified))

# %%
print(stringified.decode('utf-8'))

# %%
for nb, line in enumerate(open('data/sonnets/116.txt')):
    node = lxml.etree.Element('line')
    node.attrib['n'] = str(nb + 1)
    node.text = line.strip()
    root.append(node)
    # voltas typically, but not always occur between the octave and sextet
    if nb == 8:
        node = lxml.etree.Element('volta')
        root.append(node)

# %%
print(lxml.etree.tostring(tree, pretty_print=True).decode())

# %%
# Loop over all nodes in the tree
for node in root:
    # Leave the volta node alone. A continue statement instructs
    # Python to move on to the next item in the loop.
    if node.tag == 'volta':
        continue
    # We chop off and store verse-final punctuation:
    punctuation = ''
    if node.text[-1] in ',:;.':
        punctuation = node.text[-1]
        node.text = node.text[:-1]
    # Make a list of words using the split method
    words = node.text.split()
    # We split rhyme words and other words:
    other_words, rhyme = words[:-1], words[-1]
    # Replace the node's text with all text except the rhyme word
    node.text = ' '.join(other_words) + ' '
    # We create the rhyme element, with punctuation (if any) in its tail
    elt = lxml.etree.Element('rhyme')
    elt.text = rhyme
    elt.tail = punctuation
    # We add the rhyme to the line:
    node.append(elt)

tree = lxml.etree.ElementTree(root)
print(lxml.etree.tostring(tree, pretty_print=True).decode())

# %%
with open('data/sonnets/116.xml', 'w') as f:
    f.write(
        lxml.etree.tostring(
            root, xml_declaration=True, pretty_print=True, encoding='utf-8').decode())

# %%
root = lxml.etree.Element('sonnet')
# Add an author attribute to the root node
root.attrib['author'] = 'William Shakespeare'
# Add a year attribute to the root node
root.attrib['year'] = '1609'

for nb, line in enumerate(open('data/sonnets/116.txt')):
    line_node = lxml.etree.Element('line')
    # Add a line number attribute to each line node
    line_node.attrib['n'] = str(nb + 1)

    # Make different nodes for words and non-words
    word = ''
    for char in line.strip():
        if char.isalpha():
            word += char
        else:
            word_node = lxml.etree.Element('w')
            word_node.text = word
            line_node.append(word_node)
            word = ''

            char_node = lxml.etree.Element('c')
            char_node.text = char
            line_node.append(char_node)

    # don't forget last word:
    if word:
        word_node = lxml.etree.Element('w')
        word_node.text = word
        line_node.append(word_node)

    rhyme_node = lxml.etree.Element('rhyme')
    # We use xpath to find the final w-element in the line
    # and wrap it in a line element
    rhyme_node.append(line_node.xpath('//w')[-1])
    line_node.replace(line_node.xpath('//w')[-1], rhyme_node)

    root.append(line_node)

    # Add the volta node
    if nb == 8:
        node = lxml.etree.Element('volta')
        root.append(node)

tree = lxml.etree.ElementTree(root)
xml_string = lxml.etree.tostring(tree, pretty_print=True).decode()
# Print a snippet of the tree:
print(xml_string[:xml_string.find("</line>") + 8] + '  ...')

# %%
tree = lxml.etree.parse('data/folger/xml/Oth.xml')
print(tree.getroot().find('.//{http://www.tei-c.org/ns/1.0}title').text)

# %%
print(tree.getroot().find('title'))

# %%
NSMAP = {'tei': 'http://www.tei-c.org/ns/1.0'}
print(tree.getroot().find('.//tei:title', namespaces=NSMAP).text)

# %%
import bs4 as bs

html_doc = """

    <title>Henry IV, Part I</title>

KING

        FTLN 0001
        So shaken as we are, so wan with care,

        FTLN 0002
        Find we a time for frighted peace to pant

        FTLN 0003
        And breathe short-winded accents of new broils

        FTLN 0004
        To be commenced in strands afar remote.

"""

html = bs.BeautifulSoup(html_doc, 'html.parser')

# %%
# print the documents <title> (from head)
print(html.title)

# %%
# print the first
 element and its content
print(html.p)

# %%
# print the text of a particular element, e.g. the <title>
print(html.title.text)

# %%
# print the parent tag (and its content) of the first
 element
print(html.p.parent)

# %%
# print the parent tag name of the first
 element
print(html.p.parent.name)

# %%
# find all occurrences of the  element
print(html.find_all('a'))

# %%
# find a
 element with a specific ID
print(html.find('p', id='line-1.1.3'))

# %%
def html2txt(fpath):
    """Convert text from a HTML file into a string.

    Arguments:
        fpath: a string pointing to the filename of the HTML file

    Returns:
        A string representing the text extracted from the supplied
        HTML file.

    """
    with open(fpath) as f:
        html = bs.BeautifulSoup(f, 'html.parser')
    return html.get_text()

# %%
fp = 'data/folger/html/1H4.html'
text = html2txt(fp)
start = text.find('Henry V')
print(text[start:start + 500])

# %%
with open(fp) as f:
    html = bs.BeautifulSoup(f, 'html.parser')
toc = html.find('table', attrs={'class': 'contents'})

# %%
def toc_hrefs(html):
    """Return a list of hrefs from a document's table of contents."""
    toc = html.find('table', attrs={'class': 'contents'})
    hrefs = []
    for tr in toc.find_all('tr'):
        for td in tr.find_all('td'):
            for a in td.find_all('a'):
                hrefs.append(a.get('href'))
    return hrefs

# %%
items = toc_hrefs(html)
print(items[:5])

# %%
def get_href_div(html, href):
    """Retrieve the
 element corresponding to the given href."""
    href = href.lstrip('#')
    div = html.find('div', attrs={'id': href})
    if div is None:
        div = html.find('a', attrs={'name': href}).findNext('div')
    return div

# %%
def html2txt(fname, concatenate=False):
    """Convert text from a HTML file into a string or sequence of strings.

    Arguments:
        fpath: a string pointing to the filename of the HTML file.
        concatenate: a boolean indicating whether to concatenate the
            extracted texts into a single string. If False, a list of
            strings representing the individual sections is returned.

    Returns:
        A string or list of strings representing the text extracted
        from the supplied HTML file.

    """
    with open(fname) as f:
        html = bs.BeautifulSoup(f, 'html.parser')
    # Use a concise list comprehension to create the list of texts.
    # The same list could be constructed using an ordinary for-loop:
    #    texts = []
    #    for href in toc_hrefs(html):
    #        text = get_href_div(html, href).get_text()
    #        texts.append(text)
    texts = [get_href_div(html, href).get_text() for href in toc_hrefs(html)]
    return '\n'.join(texts) if concatenate else texts

# %%
texts = html2txt(fp)
print(texts[6][:200])

# %%
import urllib.request

page = urllib.request.urlopen('https://en.wikipedia.org/wiki/William_Shakespeare')
html = page.read()

# %%
import bs4

soup = bs4.BeautifulSoup(html, 'html.parser')
print(soup.get_text().strip()[:300])

# %%
import re

for script in soup(['script', 'style']):
    script.extract()
text = soup.get_text()
text = re.sub('\s*\n+\s*', '\n', text)  # remove multiple linebreaks:
print(text[:300])

# %%
links = soup.find_all('a')
print(links[9].prettify())

# %%
V = {1, 2, 3, 4, 5}
E = {(1, 2), (1, 4), (2, 5), (3, 4), (4, 5)}

# %%
import networkx as nx

G = nx.Graph()
G.add_nodes_from(V)
G.add_edges_from(E)

# %%
import matplotlib.pyplot as plt

nx.draw_networkx(G, font_color="white")
plt.axis('off');

# %%
NSMAP = {'tei': 'http://www.tei-c.org/ns/1.0'}

def character_network(tree):
    """Construct a character interaction network.

    Construct a character interaction network for Shakespeare texts in
    the Folger Digital Text collection. Character interaction networks
    are constructed on the basis of successive speaker turns in the texts,
    and edges between speakers are created when their utterances follow
    one another.

    Arguments:
        tree: An lxml.ElementTree instance representing one of the XML
            files in the Folger Shakespeare collection.

    Returns:
        A character interaction network represented as a weighted,
        undirected NetworkX Graph.

    """
    G = nx.Graph()
    # extract a list of speaker turns for each scene in a play
    for scene in tree.iterfind('.//tei:div2[@type="scene"]', NSMAP):
        speakers = scene.findall('.//tei:sp', NSMAP)
        # iterate over the sequence of speaker turns...
        for i in range(len(speakers) - 1):
            # ... and extract pairs of adjacent speakers
            try:
                speaker_i = speakers[i].attrib['who'].split('_')[0].replace('#', '')
                speaker_j = speakers[i + 1].attrib['who'].split('_')[0].replace('#', '')
                # if the interaction between two speakers has already
                # been attested, update their interaction count
                if G.has_edge(speaker_i, speaker_j):
                    G[speaker_i][speaker_j]['weight'] += 1
                # else add an edge between speaker i and j to the graph
                else:
                    G.add_edge(speaker_i, speaker_j, weight=1)
            except KeyError:
                continue
    return G

# %%
tree = lxml.etree.parse('data/folger/xml/Ham.xml')
G = character_network(tree.getroot())

# %%
print(f"N nodes = {G.number_of_nodes()}, N edges = {G.number_of_edges()}")

# %%
import collections

interactions = collections.Counter()

for speaker_i, speaker_j, data in G.edges(data=True):
    interaction_count = data['weight']
    interactions[speaker_i] += interaction_count
    interactions[speaker_j] += interaction_count

nodesizes = [interactions[speaker] * 5 for speaker in G]

# %%
# Create an empty figure of size 15x15
fig = plt.figure(figsize=(15, 15))
# Compute the positions of the nodes using the spring layout algorithm
pos = nx.spring_layout(G, k=0.5, iterations=200)
# Then, add the edges to the visualization
nx.draw_networkx_edges(G, pos, alpha=0.4)
# Subsequently, add the weighted nodes to the visualization
nx.draw_networkx_nodes(G, pos, node_size=nodesizes, alpha=0.4)
# Finally, add the labels (i.e. the speaker IDs) to the visualization
nx.draw_networkx_labels(G, pos, fontsize=14)
plt.axis('off');

# %%
from copy import deepcopy
G0 = deepcopy(G)

for u, v, d in G0.edges(data=True):
    d['weight'] = 1

nodesizes = [interactions[speaker] * 5 for speaker in G0]

fig = plt.figure(figsize=(15, 15))
pos = nx.spring_layout(G0, k=0.5, iterations=200)
nx.draw_networkx_edges(G0, pos, alpha=0.4)
nx.draw_networkx_nodes(G0, pos, node_size=nodesizes, alpha=0.4)
nx.draw_networkx_labels(G0, pos, fontsize=14)
plt.axis('off');

# %%
G0.remove_node('Hamlet')

# %%
fig = plt.figure(figsize=(15, 15))
pos = nx.spring_layout(G0, k=0.5, iterations=200)
nodesizes = [interactions[speaker] * 5 for speaker in G0]
nx.draw_networkx_edges(G0, pos, alpha=0.4)
nx.draw_networkx_nodes(G0, pos, node_size=nodesizes, alpha=0.4)
nx.draw_networkx_labels(G0, pos, fontsize=14)
plt.axis('off');

# %%
import json
from networkx.readwrite import json_graph

with open('hamlet.json', 'w') as f:
    json.dump(json_graph.node_link_data(G), f)

with open('hamlet.json') as f:
    d = json.load(f)

G = json_graph.node_link_graph(d)
print(f"Graph with {len(G.nodes())} nodes and {len(G.edges())} edges.")

# %%
# Undocumented code snippet used in chapter (e.g., for figure generation)
import functools
from copy import deepcopy
G1 = deepcopy(G)

for u, v, d in G.edges(data=True):
    if d["weight"] < 10:
        G1.remove_edge(u, v)

G1 = nx.relabel_nodes(G1, {"SOLDIERS.FORTINBRAS.Captain": "Fortinbras.Captain"})
# rename verbose name for Fortinbras' Captain
#SOLDIERS.FORTINBRAS.Captain

subgraphs = [G1.subgraph(c).copy() for c in nx.connected_components(G1)]
# functools.reduce is similar to foldl in Haskell and fold_left in OCaml
def larger_graph(graph1, graph2):
    return graph2 if len(graph2.nodes()) > len(graph1.nodes()) else graph1
G1 = functools.reduce(larger_graph, subgraphs, subgraphs[0])

fig = plt.figure(figsize=(9, 6))
pos = nx.spring_layout(G1, k=0.5, iterations=2000, seed=1)
nx.draw_networkx_edges(G1, pos, alpha=0.4)
nx.draw_networkx_nodes(G1, pos, node_size=[degree * 100 for _, degree in G1.degree()], alpha=0.4)
nx.draw_networkx_labels(G1, pos, fontsize=14)
plt.axis('off')
plt.tight_layout()
plt.savefig('img/hamlet-minimum-10-interactions.png')


#!/usr/bin/env python3

import time
import webbrowser
import random
from random import shuffle
import sys

from bs4 import BeautifulSoup
import urllib.request

finalSearch = True
page = "https://en.wikipedia.org/wiki/Special:Random"
#page = "https://en.wikipedia.org/wiki/Michael_D._Higgins"
hops = 0
firstSearch = True

if(len(sys.argv) > 1):
    mode = sys.argv[1]
else:
    mode = "search"

if(mode == "search"):
    while(finalSearch):

        resp = urllib.request.urlopen(page)
        soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))

        if(firstSearch):
            firstSearch = False
            print("\n\nStarting Page: " + soup.title.string)

        links = soup.find_all('a', href=True)

        badLink = True
        link = ""
        hops += 1

        #favor the desired page
        for i in links:
            if(i.get('href') == "/wiki/Halloween"):
                link = i
                badLink = False
                break

        if(badLink):
            #favor the primary
            primaryLinks = [line.rstrip('\n') for line in open("primary.txt")]
            shuffle(primaryLinks)
            for i in links:
                for linkED in primaryLinks:
                    if(str(i.get('title')).replace(" ", "_") == linkED):
                        print("\nGoing to primary page: " + i.get("title"))
                        link = i
                        badLink = False
                        break

        if(badLink):
            #favor the secondary
            secondaryLinks = [line.rstrip('\n') for line in open("secondary.txt")]
            shuffle(secondaryLinks)
            for i in links:
                for linkED in secondaryLinks:
                    if(str(i.get('title')).replace(" ", "_") == linkED):
                        print("\nGoing to secondary page: " + i.get("title"))
                        link = i
                        badLink = False
                        break

        while(badLink):
            link = random.choice(links)

            if(link.get('href') == None or link.get('href') == ""):
                continue

            if(link.get('href')[:5] == "https"):
                continue

            if(link.get('href')[:5] != "/wiki"):
                continue

            if(link.get('title') == None or link.get('title') == ""):
                continue

            if(link.get('href') == "/wiki/Main_Page"):
                continue

            if(":" in link.get('href')):
                continue

            try:
               link.get('href')
            except:
                continue

            badLink = False

        if(link.get('href') == "/wiki/Halloween"):
            print("\n\nWE FOUND HALLOWEEN IN " + str(hops) + " HOPS")
            time.sleep(2)
            webbrowser.open("https://www.youtube.com/watch?v=PBHvJrVma8Y&feature=youtu.be&t=1s")
            time.sleep(1)
            webbrowser.open("https://en.wikipedia.org" + link.get('href'))
            exit()

        print("\n" + str(link.get('title').encode("cp437", "ignore").strip().decode('cp437')))
        print("https://en.wikipedia.org" + str(link.get('href').encode("cp437", "ignore").strip().decode('cp437')))
        page = "https://en.wikipedia.org" + link.get('href')
elif(mode == "primary"):
    try:
        file = open("primary.txt", 'r')
    except FileNotFoundError:
        file = open("primary.txt", 'w')

    primaryLinks = [line.rstrip('\n') for line in open("primary.txt")]

    for x in range(0, 50000):
        resp = urllib.request.urlopen(page)
        soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))

        links = soup.find_all('a', href=True)

        for i in links:
            if(i.get('href') == "/wiki/Halloween"):
                if(soup.title.string not in links):
                    title = soup.title.string
                    title = title[:-12]
                    title = title.replace(" ", "_")
                    print("Adding: " + title.encode("cp437", "ignore").strip().decode('cp437'))
                    with open("primary.txt", "a") as myfile:
                        myfile.write(title  + "\n")
                    break

elif (mode == "secondary"):
    try:
        file = open("secondary.txt", 'r')
    except FileNotFoundError:
        file = open("secondary.txt", 'w')

    primaryLinks = [line.rstrip('\n') for line in open("primary.txt")]
    secondaryLinks = [line.rstrip('\n') for line in open("secondary.txt")]

    for x in range(0, 50000):
        resp = urllib.request.urlopen(page)
        soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))

        links = soup.find_all('a', href=True)

        for i in links:
            for y in primaryLinks:
                title = "The_Troubles"
                if(i.get('title') == y):
                    title = soup.title.string
                    title = title[:-12]
                    title = title.replace(" ", "_")
                if(title not in secondaryLinks):
                    print("Adding: " + title.encode("cp437", "ignore").strip().decode('cp437'))
                    with open("secondary.txt", "a") as myfile:
                        myfile.write(title  + "\n")
                    break


class Solution:

    # Union-Find [O(n2): 30%]
    def longestConsecutive(self, nums: List[int]) -> int:
        if not nums:
            return 0

        numToIdx = collections.defaultdict(int)
        parents = [-1] * len(nums)
        self.count = len(nums)

        for i in range(len(nums)):
            if nums[i] in numToIdx:  #avoid redandunts
                continue
            if nums[i] + 1 in numToIdx:
                self.union(parents, i, numToIdx[nums[i] + 1])
            if nums[i] - 1 in numToIdx:
                self.union(parents, i, numToIdx[nums[i] - 1])
            numToIdx[nums[i]] = i
        print(numToIdx)
        print(parents)
        return -min(parents)

    def union(self, parents, a, b):
        headA = self.find(parents, a)  #index
        headB = self.find(parents, b)  #index
        if headA != headB:
            countA = - parents[headA] if parents[headA] < 0  else 1
            countB = - parents[headB] if parents[headB] < 0  else 1
            parents[headA] = headB
            parents[headB] = - (countA + countB)   #count stored in head

    def find(self, parents, idx):
        if parents[idx] < 0:
            return idx
        return self.find(parents, parents[idx])


import pandas
import pandas as pd
import numpy as np
import datetime

SHEET_NAME = "savedrecs"
YEAR = int(datetime.date.today().year)
dataframe = ""

#Pls fill the absolute path
WOS_EXCEL_PATH = "C:\\Users\itaug\Desktop\\testfyp\wos_result_411.xlsx"

def revise_WOS_output():
    revise_WOS_dataframe(WOS_EXCEL_PATH)
    # Output to the excel file
    dataframe.reset_index(drop=True, inplace=True)
    dataframe.to_excel("revise_wos_result.xlsx", columns=["Author Full Names", "Article Title", "Publication Year",
                                               "Times Cited, WoS Core", "Cited Reference Count", "Abstract",
                                               "Publisher"], index=True, sheet_name='savedrecs')

def revise_WOS_dataframe(WOS_EXCEL_PATH):
    global dataframe
    dataframe = pd.read_excel(WOS_EXCEL_PATH, sheet_name=SHEET_NAME)
    cite_mean = dataframe['Times Cited, WoS Core'].mean()

    # define the value of "time cited" are their over the mean
    dataframe['Count_Cited'] = np.where(dataframe['Times Cited, WoS Core'] < round(cite_mean), 0, 1)

    # if publish year are empty == 2022
    dataframe['Publication Year'] = np.where(np.isnan(dataframe['Publication Year']), YEAR,
                                             dataframe['Publication Year'])

    # chose the columns ['Count_Cited'] == 1
    dataframe = dataframe[dataframe['Count_Cited'].isin([1])].sort_values(by='Publication Year')

    # save the abstract to list
    # abstract_list = dataframe["Abstract"].tolist()

    return dataframe

def modify_WOS_dataframe(WOS_EXCEL_PATH):
    global dataframe
    dataframe = pd.read_excel(WOS_EXCEL_PATH, sheet_name=SHEET_NAME)
    return dataframe

def modify_VOS_dataframe(VOS_EXCEL_PATH):
    global dataframe
    dataframe = pd.read_excel(VOS_EXCEL_PATH, sheet_name=SHEET_NAME)
    return dataframe

def pdf_dataframe(PDF_EXCEL_PATH):
    global dataframe
    dataframe = pd.read_excel(PDF_EXCEL_PATH, sheet_name='my dataframe')
    return dataframe

if __name__ == "__main__":
    revise_WOS_output()


import numpy
import psycopg2

import lib.fits
import lib.misc
import lib.random_algos
import lib.dbtable
import lib.sourcetable
import lib.common
import lib.config
from lib.misc import PoppingOrderedDict

if lib.config.MULTICORE:
    from lib import pipe_printf

import glob
import io
import itertools
import os
import re
import sys
import warnings

def main():
    import argparse
    parser = argparse.ArgumentParser(
        fromfile_prefix_chars='@',
        description='Read a rerun directory to create "random" summary table.')

    parser.add_argument('rerunDir', help="Rerun directory from which to read data")
    parser.add_argument('schemaName', help="DB schema name in which to load data")

    parser.add_argument("--table-name", default="random", help="Top-level table's name")
    parser.add_argument("--table-space", default="", help="DB table space for tables")
    parser.add_argument("--index-space", default="", help="DB table space for indexes")
    parser.add_argument("--db-server", metavar="key=value", nargs="+", action="append", help="DB to connect to. This option must come later than non-optional arguments.")

    parser.add_argument("--with-skymap-wcs", action="store",
        help="""Use skymap_wcs at the specified path instead of calexp-*.fits.
            To generate skymap_wcs, run 'generate-skymap_wcs.py skyMap.pickle'
        """
    )

    parser.add_argument('--create-index',  action='store_true',
       help="Create index (only; don't insert data)")

    args = parser.parse_args()

    if args.db_server:
        lib.config.dbServer.update(keyvalue.split('=', 1) for keyvalue in itertools.chain.from_iterable(args.db_server))

    lib.config.tableSpace = args.table_space
    lib.config.indexSpace = args.index_space
    lib.config.withSkymapWcs = args.with_skymap_wcs

    filters = lib.common.get_existing_filters(args.rerunDir)
    if args.create_index:
        create_index_on_mastertable(args.rerunDir, args.schemaName, filters)
    else:
        create_mastertable_if_not_exists(args.rerunDir, args.schemaName, args.table_name, filters)
        insert_into_mastertable(args.rerunDir, args.schemaName, args.table_name, filters)

def create_mastertable_if_not_exists(rerunDir, schemaName, masterTableName, filters):
    """
    Create the master table if it does not exist.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param masterTableName
        Name of the master table
    @param filters
        List of filter names
    """
    bNeedCreating = False

    db = lib.common.new_db_connection()
    with db.cursor() as cursor:
        try:
            cursor.execute('SELECT 0 FROM "{schemaName}"."{masterTableName}" WHERE FALSE;'.format(**locals()))
        except psycopg2.ProgrammingError:
            bNeedCreating = True
            db.rollback()

    if bNeedCreating:
        with db.cursor() as cursor:
            cursor.execute('CREATE SCHEMA IF NOT EXISTS "{schemaName}"'.format(**locals()))
            create_mastertable(cursor, rerunDir, schemaName, masterTableName, filters)
        db.commit()
    else:
        db.close()
        drop_index_from_mastertable(rerunDir, schemaName, filters)

def create_mastertable(cursor, rerunDir, schemaName, masterTableName, filters):
    """
    Create the master table.
    The master table will actually be a view, which JOINs (not UNIONs) child tables.
    The child tables will also be created during a call to this function.

    This function only creates these tables. It does not insert data into them.

    @param cursor
        DB connection's cursor object
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param masterTableName
        Name of the master table
    @param filters
        List of filter names
    """
    tract, patch, filter = get_an_exisiting_catalog_id(rerunDir)
    catPath = get_catalog_path(rerunDir, tract, patch, filter)
    multibands, object_id, coord = get_catalog_schema_from_file(catPath, None)

    for table in multibands.values():
        table.set_filters(filters)

    for table in multibands.values():
        table.transform(rerunDir, tract, patch, filter, coord)

    # Create source tables
    for table in multibands.values():
        table.create(cursor, schemaName)

    # Create master table
    commentOnTable = """
    Catalog of random objects.

    Notice that the object_id in this table do not agree with those in the other tables including
    forced, meas and specz. This means it is totally nonsense to join
    random to these tables: <s>forced JOIN random USING(object_id)</s>, for example.
    """

    # Because the number of columns exceeds PostgreSQL's limit,
    # we divide the master table into "random", "random2", "random3", ...

    listMultibands = [ multibands ]
    # listMultibands.append(multibands.pop_many([
    #     "...", "...", ...
    # ]))

    for iPart, multibands in enumerate(listMultibands, start=1):
        dbTables = [table.name for table in multibands.values()]
        if len(dbTables) > 1:
            dbTables = [ '"{}"."{}"'.format(schemaName, dbTables[0]) ] + [
                'LEFT JOIN "{}"."{}" USING (object_id)'.format(schemaName, table)
                for table in dbTables[1:]
            ]
            dbTables = """
            """.join(dbTables)
        else:
            dbTables = '"{}"."{}"'.format(schemaName, dbTables[0])

        fieldDefs = []
        for table in multibands.values():
            fieldDefs += table.get_bandindependent_exported_fields()
        for filter in filters:
            for table in multibands.values():
                fieldDefs += table.get_exported_fields(filter)

        fieldDefs.insert(0, ("object_id", "object_id", "",
            "Unique ID in 64bit integer. Be careful not to have it converted to a 32bit integer or 64bit floating point."
        ))

        sFieldDefs = """,
        """.join(
            '{} AS {}'.format(definition, name) for name, definition, unit, doc in fieldDefs
        )

        tableName = "{masterTableName}{iPart}".format(**locals()) if iPart > 1 else masterTableName

        cursor.execute("""
        CREATE VIEW "{schemaName}"."{tableName}" AS (
            SELECT
                {sFieldDefs}
            FROM
                {dbTables}
        )
        """.format(**locals())
        )

        for name, definition, unit, doc in fieldDefs:
            cursor.execute("""
            COMMENT ON COLUMN "{schemaName}"."{tableName}"."{name}" IS %(comment)s
            """.format(**locals()), dict(comment = "{doc} || {unit}".format(**locals()))
            )

        cursor.execute("""
        COMMENT ON VIEW "{schemaName}"."{tableName}" IS %(comment)s
        """.format(**locals()), dict(comment = commentOnTable)
        )

def insert_into_mastertable(rerunDir, schemaName, masterTableName, filters):
    """
    Insert data into the master table.
    The data will actually flow not into the master table but into its children.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param masterTableName
        Name of the master table
    @param filters
        List of filter names
    """
    for tract in lib.common.get_existing_tracts(rerunDir):
        for patch in get_existing_patches(rerunDir, tract):
            insert_patch_into_mastertable(rerunDir, schemaName, masterTableName, filters, tract, patch)

def insert_patch_into_mastertable(rerunDir, schemaName, masterTableName, filters, tract, patch):
    """
    Insert a specific patch into the master table.
    The data will actually flow not into the master table but into its children.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param masterTableName
        Name of the master table
    @param filters
        List of filter names
    @param tract
        Tract number.
    @param patch
        Patch number (x*100 + y)
    """
    catPaths = {}
    for filter in filters:
        catPath = get_catalog_path(rerunDir, tract, patch, filter)
        if lib.common.path_exists(catPath):
            catPaths[filter] = catPath

    if not catPaths:
        return

    db = lib.common.new_db_connection()
    with db.cursor() as cursor:
        if is_patch_already_inserted(cursor, schemaName, tract, patch, catPaths.keys()):
            lib.misc.warning("Skip because already inserted: (tract,patch) = ({tract}, {patch})".format(**locals()))
            return

        object_id = None
        multibands = {}
        for filter, catPath in catPaths.items():
            if object_id is None:
                mult, object_id, coord = get_catalog_schema_from_file(catPath, None)
            else:
                mult, dummy, dummy2 = get_catalog_schema_from_file(catPath, object_id)

            for table in mult.values():
                table.transform(rerunDir, tract, patch, filter, coord)

                if table.name not in multibands:
                    multibands[table.name] = []
                multibands[table.name].append((table, filter))

        for tables in multibands.values():
            insert_patch_into_multibandtable(cursor, schemaName, tables, object_id)

    db.commit()

def insert_patch_into_multibandtable(cursor, schemaName, tables, object_id):
    """
    Insert a patch into a multiband table.
    @param cursor
        DB connection's cursor object
    @param schemaName
        Name of the schema in which to locate the master table
    @param tables
        List of (table: DBTable, filter: str).
        Tables in this list must have identical object_id (Or, identical (tract, patch))
        with different colors.
    @param object_id
        numpy.array of object ID. This is used as the primary key.
    """
    columns = [ object_id ]
    fieldNames = [ "object_id" ]
    format = "%ld"

    for name, fmt, cols in tables[0][0].get_bandindependent_backend_field_data():
        columns.extend(cols)
        fieldNames.append(name)
        format += "\t" + fmt

    for table, filter in tables:
        for name, fmt, cols in table.get_backend_field_data(filter):
            columns.extend(cols)
            fieldNames.append(name)
            format += "\t" + fmt

    format += "\n"
    format = format.encode("utf-8")

    if lib.config.MULTICORE:
        fin = pipe_printf.open(format, *columns)
        cursor.copy_from(fin, '"{}"."{}"'.format(schemaName, table.name), sep='\t', columns=fieldNames)
    else:
        tsv = b''.join(format % tpl for tpl in zip(*columns))
        fin = io.BytesIO(tsv)
        cursor.copy_from(fin, '"{}"."{}"'.format(schemaName, table.name), sep='\t', size=-1, columns=fieldNames)

def create_index_on_mastertable(rerunDir, schemaName, filters):
    """
    Create indexes on the master table.
    The indexes will actually be set on the master's children.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param filters
        List of filter names
    """
    tract, patch, filter = get_an_exisiting_catalog_id(rerunDir)
    catPath = get_catalog_path(rerunDir, tract, patch, filter)

    multibands, object_id, coord = get_catalog_schema_from_file(catPath, None)

    for table in multibands.values():
        table.set_filters(filters)

    db = lib.common.new_db_connection()
    with db.cursor() as cursor:
        for table in multibands.values():
            table.create_index(cursor, schemaName)
    db.commit()

def drop_index_from_mastertable(rerunDir, schemaName, filters):
    """
    Drop indexes from the master table.
    The indexes to be dropped has actually been set on the master's children.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param schemaName
        Name of the schema in which to locate the master table
    @param filters
        List of filter names
    """
    tract, patch, filter = get_an_exisiting_catalog_id(rerunDir)
    catPath = get_catalog_path(rerunDir, tract, patch, filter)

    multibands, object_id, coord = get_catalog_schema_from_file(catPath, None)

    for table in multibands.values():
        table.set_filters(filters)

    db = lib.common.new_db_connection()
    with db.cursor() as cursor:
        for table in multibands.values():
            table.drop_index(cursor, schemaName)
    db.commit()

def get_catalog_schema_from_file(path, object_id):
    """
    Get fields in a "ran-*.fits" file.
    @param path
        Path to a "ran-*.fits" file
    @param object_id
        numpy.array of object_id. Optional.
    @return (multibands, object_id, coord)
        * multibands: PoppingOrderedDict mapping name: str -> table: DBTable
        * object_id: numpy.array of object_id.
        * coord: {"ra": numpy.array, "dec": numpy.array},
            in which angles are in degrees.
    """
    table = lib.sourcetable.SourceTable.from_hdu(lib.fits.fits_open(path)[1])

    these_object_id = table.cutout_subtable("id").fields["id"].data

    if (object_id is not None) and (not numpy.all(these_object_id == object_id)):
        raise RuntimeError("object_id in ran-*.fits doesn't agree with another filter" + path)

    algos = PoppingOrderedDict(
        (name, algoclass(table))
        for name, algoclass in lib.random_algos.random_algos.items()
    )

    coord = algos["random_coord"].coord

    # To suppress warnings "Ignored field: ...",
    # ignore the fields explicitly.
    def ignore(prefix):
        table.cutout_subtable(prefix)

    for name in lib.random_algos.random_algos_ignored:
        ignore(name)

    for field in table.fields:
        lib.misc.warning('Ignored field: ', field, 'in', path)

    dbtables = PoppingOrderedDict()
    dbtables["_random:random"] = DBTable_Random("_random:random", algos)

    return dbtables, these_object_id, coord

class DBTable_Random(lib.dbtable.DBTable):
    def create_index(self, cursor, schemaName):
        lib.dbtable.DBTable.create_index(self, cursor, schemaName)

        indexSpace = lib.config.get_index_space()

        cursor.execute("""
        CREATE INDEX
            "{self.name}_skymap_id_idx"
        ON
            "{schemaName}"."{self.name}"
            ( public.skymap_from_object_id(object_id)
            )
        {indexSpace}
        """.format(**locals())
        )
        cursor.execute("""
        CREATE INDEX
            "{self.name}_coord_idx"
        ON
            "{schemaName}"."{self.name}"
        USING GiST
            ( coord
            )
        {indexSpace}
        WHERE
            coord IS NOT NULL
        """.format(**locals())
        )

        # indices WHERE isprimary = True

        cursor.execute("""
        CREATE UNIQUE INDEX
            "{self.name}_object_id_primary_idx"
        ON
            "{schemaName}"."{self.name}"
            ( object_id
            )
        {indexSpace}
        WHERE
          isprimary
        """.format(**locals())
        )
        cursor.execute("""
        CREATE INDEX
            "{self.name}_skymap_id_primary_idx"
        ON
            "{schemaName}"."{self.name}"
            ( public.skymap_from_object_id(object_id)
            )
        {indexSpace}
        WHERE
          isprimary
        """.format(**locals())
        )
        cursor.execute("""
        CREATE INDEX
            "{self.name}_coord_primary_idx"
        ON
            "{schemaName}"."{self.name}"
        USING GiST
            ( coord
            )
        {indexSpace}
        WHERE
            coord IS NOT NULL
            AND isprimary
        """.format(**locals())
        )

    def drop_index(self, cursor, schemaName):
        lib.dbtable.DBTable.drop_index(self, cursor, schemaName)

        cursor.execute("""
        DROP INDEX IF EXISTS
            "{schemaName}"."{self.name}_skymap_id_idx"
        """.format(**locals())
        )
        cursor.execute("""
        DROP INDEX IF EXISTS
            "{schemaName}"."{self.name}_coord_idx"
        """.format(**locals())
        )

        # indices WHERE isprimary = True

        cursor.execute("""
        DROP INDEX IF EXISTS
            "{schemaName}"."{self.name}_object_id_primary_idx"
        """.format(**locals())
        )
        cursor.execute("""
        DROP INDEX IF EXISTS
            "{schemaName}"."{self.name}_skymap_id_primary_idx"
        """.format(**locals())
        )
        cursor.execute("""
        DROP INDEX IF EXISTS
            "{schemaName}"."{self.name}_coord_primary_idx"
        """.format(**locals())
        )

    def create(self, cursor, schemaName):
        """
        Create table in the database.
        A primary key "object_id" will be included
        regardless of whether it exists in self.fields or not.
        @param cursor
            DB connection's cursor object
        @param schemaName
            Name of the schema in which to locate the master table
        """
        members = ["object_id Bigint"]

        # band-independent fields
        for name, type in self.algos['random_coord'].get_backend_fields(""):
            members.append("{name}  {type}".format(**locals()))

        # multiband fields
        for filter in self.filters:
            filt = lib.common.filterToShortName[filter] + "_" if filter else ""
            for key, algo in self.algos.items():
                if key == 'random_coord': continue
                for name, type in algo.get_backend_fields(filt):
                    members.append("{name}  {type}".format(**locals()))

        members = """,
        """.join(members)

        tableSpace = lib.config.get_table_space()

        cursor.execute("""
        CREATE TABLE "{schemaName}"."{self.name}" (
            {members}
        )
        {tableSpace}
        """.format(**locals())
        )

    def get_bandindependent_backend_field_data(self):
        """
        Get band-independent field data for the backend table.
        @return list of (fieldname, printf_format, [column]).
            'column' is a numpy.array. An example of the return value is:
            ("i_point", "(%.16e,%.16e)", [x, y]),
            in which x and y are numpy.array.
        """
        return self.algos['random_coord'].get_backend_field_data("")

    def get_backend_field_data(self, filter):
        """
        Get field data for the backend table.
        @param filter (str)
            Filter name.
        @return list of (fieldname, printf_format, [column]).
            'column' is a numpy.array. An example of the return value is:
            ("i_point", "(%.16e,%.16e)", [x, y]),
            in which x and y are numpy.array.
        """
        filt = lib.common.filterToShortName[filter] + "_" if filter else ""
        members = []

        for key, algo in self.algos.items():
            if key == 'random_coord': continue
            members += algo.get_backend_field_data(filt)

        return members

    def get_bandindependent_exported_fields(self):
        """
        @return list of (fieldname, definition, unit, document).
            Each field can be exported as:
                {definition} AS {fieldname}.
        """
        return self.algos['random_coord'].get_frontend_fields("")

    def get_exported_fields(self, filter):
        """
        Get field data for the frontend view.
        @param filter (str)
            Filter name.
        @return list of (fieldname, definition, unit, document).
            Each field can be exported as:
                {definition} AS {fieldname}.
        """
        filt = lib.common.filterToShortName[filter] + "_" if filter else ""
        members = []

        for key, algo in self.algos.items():
            if key == 'random_coord': continue
            members += algo.get_frontend_fields(filt)

        return members

def get_existing_patches(rerunDir, tract):
    """
    Search "rerunDir" and return existing patches in the given tract.
    @param rerunDir
        Path to the rerun directory from which to generate the master table
    @param tract
        Tract number.
    @return
        List of tract numbers, sorted.
    """

    patches = set(
        os.path.basename(os.path.dirname(path))
        for path in glob.iglob("{rerunDir}/deepCoadd-results/*/{tract}/*,*/ran-*.fits*".format(**locals()))
        if path.endswith('.fits') or path.endswith('.fits.gz')
    )

    return sorted(set(
        lib.common.patch_to_number(patch) for patch in patches if re.match(r'^[0-9]+,[0-9]+$', patch)
    ))

def get_catalog_path(rerunDir, tract, patch, filter):
    """
    Get the path to the "ran-*.fits" catalog identified by the arguments.
    The file may be compressed, but the path is virtualized so it always ends with '.fits'.
    """
    if patch == "*":
        x, y = "*", "*"
    else:
        x, y = patch // 100, patch % 100
    return "{rerunDir}/deepCoadd-results/{filter}/{tract}/{x},{y}/ran-{filter}-{tract}-{x},{y}.fits".format(**locals())

def get_an_exisiting_catalog_id(rerunDir):
    """
    Get any one triple (tract, patch, filter) for which catalog files exist
    """
    pattern = get_catalog_path(rerunDir, "*", "*", "*")

    for imagePath in itertools.chain(glob.iglob(pattern), glob.iglob(pattern + ".gz")):
        tract, patch, filter = lib.common.path_decompose(imagePath)
        return tract, patch, filter

    raise RuntimeError("No catalog (ran-*.fits) exists.")

def is_patch_already_inserted(cursor, schemaName, tract, patch, filters):
    """
    Check whether (tract, patch, filters) has already been inserted into the DB.
    This is achieved by using a temporary table in the DB.

    This function will return
        - False if no catalogs have been inserted that match (tract, patch, *).
        - True if for every filter in filters, (tract, patch, filter) has been inserted
            and not for other filters.
        - Raise exception if there are some records that match (tract, patch, *),
            but the filter set is different from the given argument.

    @param cursor
        DB connection's cursor object
    @param schemaName
        Name of the schema in which to locate the master table
    @param tract
        Tract number
    @param patch
        Patch number (x*100 + y)
    @param filters
        List of filter names for which multiband catalogs actually exist
    """

    # file_id = (tract*10000 + patch)*100 + filter
    patchId = tract*10000 + patch
    minFileId =  patchId   *100
    maxFileId = (patchId+1)*100 - 1

    # Filter ID 0 is reserved and actual filter IDs start with 1, hence "filterOrder[f]+1"
    fileId = sorted(patchId*100 + lib.common.filterOrder[f]+1 for f in filters)

    cursor.execute("""
    CREATE TABLE IF NOT EXISTS "{schemaName}"."_temp:random_patch" (
        file_id   Bigint   PRIMARY KEY
    )
    """.format(**locals())
    )

    cursor.execute("""
    SELECT file_id FROM "{schemaName}"."_temp:random_patch" WHERE
        file_id BETWEEN {minFileId} AND {maxFileId}
    """.format(**locals())
    )

    dbFileId = sorted(id for id, in cursor)

    if fileId == dbFileId:
        return True

    if dbFileId:
        # This patch has already been registered in the DB
        # but the previous filter set was different from the current one.
        fileId = set(fileId)
        dbFileId = set(dbFileId)
        added = fileId - dbFileId
        subed = dbFileId - fileId

        toFilterName = dict((i+1, filter) for filter, i in lib.common.filterOrder.items())
        added = '(added:' + ','.join(toFilterName[i % 100] for i in added)+')' if added else ""
        subed = '(absent:' + ','.join(toFilterName[i % 100] for i in subed)+')' if subed else ""
        raise RuntimeError(
            ("Existing filters for (tract,patch)=({tract}, {patch}) has been changed. "
            + "{added}{subed}")
            .format(**locals())
        )

    sFileId = ",".join("({})".format(id) for id in fileId)

    cursor.execute("""
    INSERT INTO "{schemaName}"."_temp:random_patch"
    VALUES {sFileId}
    """.format(**locals())
    )

    return False

if __name__ == "__main__":
    main()


dia = 0

def helper(root):
    # 1. Base cases
    if not root: return 0

    # 2. Recursive calls
    lh = helper(root.left)
    rh = helper(root.right)

    # Code crux #
    dia = max(dia, lh + rh)     #/Inclase we have to calc vertices then we have to take the (1+lh+rh)

    return 1 + max(lh, rh)


from infi.gevent_utils.os import path
from infi.app_repo.errors import FilenameParsingFailed
from logging import getLogger
from re import match
logger = getLogger(__name__)

NAME = r"""(?P<package_name>[a-zA-Z]*[a-zA-Z\-_\.]+[0-9_]?[a-zA-Z\-_]+[a-zA-Z][0-9]{0,2})"""
VERSION = r"""v?(?P<package_version>(?:[\d+\.]+)(?:-develop|-[0-9\.]+(?:_g[0-9a-f]{7})?|(?:(?:\.post\d+|\.post\d+\.|\.b\d+|\.post\d+\+|\.post\d+-\d+|\.\d+\.|-[0-9\.]+-[0-9\.]+|-\d+-|-develop-\d+-|~dev\d+-\d+)(?:g[a-z0-9]{7})?))?)"""
PLATFORM = r"""(?P<platform_string>python|vmware-esx|custom|windows|aix-\d+\.\d+|solaris-\d+|linux-ubuntu-[a-z]+|linux-suse-\d+|linux-redhat-\d|linux-centos-\d|linux-rocky-\d|linux-oracle-\d|osx-\d+\.\d+|centos.el6|centos.el7|redhat.el6|redhat.el7|docker|windows-hyperv)"""
ARCHITECTURE = r"""(?P<architecture>generic|docs|sdist|x86|x64|ppc64|ppc64le|powerpc|powerpc_bff|sparc|x86_OVF10|x86_OVF10_UPDATE_ISO|x86_OVF10_UPDATE_ZIP|x64_OVF10|x64_OVF10_UPDATE_ISO|x64_OVF10_UPDATE_ZIP|x64_dd|i686|x86_64|x64_vhd)"""
EXTENSION = r"""(?P<extension>bin|rpm|deb|msi|pkg\.gz|tar\.gz|ova|vhd|iso|zip|img|exe||so|dll|pdb|cpp|exp|pak|bff)"""
TEMPLATE = r"""^{}.{}.{}.{}\.?{}$"""
FILEPATH = TEMPLATE.format(NAME, VERSION, PLATFORM, ARCHITECTURE, EXTENSION)
PLATFORM_STRING = dict(ova='vmware-esx', img='other', zip='other')
TRANSLATED_ARCHITECTURE = {"x86_64": "x64", "i686": "x86"}
TRANSLATED_PLATFORM = {"centos.el6": "linux-centos-6", "centos.el7": "linux-centos-7",
                       "redhat.el6": "linux-redhat-6", "redhat.el7": "linux-redhat-7"}
TRANSLATED_EXTENSION = {'': 'exe'}

def translate_filepath(result_tuple):
    package_name, package_version, platform_string, architecture, extension = result_tuple
    return (package_name, package_version,
            TRANSLATED_PLATFORM.get(platform_string, platform_string),
            TRANSLATED_ARCHITECTURE.get(architecture, architecture),
            TRANSLATED_EXTENSION.get(extension, extension))

def parse_filepath(filepath):
    """:returns: 5-tuple (package_name, package_version, platform_string, architecture, extension)"""
    filename = path.basename(filepath)
    result = match(FILEPATH, filename)
    if result is None:
        logger.error("failed to parse {}".format(filename))
        raise FilenameParsingFailed(filepath)
    group = result.groupdict()
    return translate_filepath((group['package_name'], group['package_version'],
                               PLATFORM_STRING.get(group['extension'], group['platform_string']),
                               group['architecture'], group['extension']))


##birthdays = {'Alice':'Apr 1','Bob':'Dec 12','Carol':'Mar 4'}
##
##while True:
##    print('Enter a name: (blank to quit)')
##    name = input()
##    if name == '':
##        break
##    if name in birthdays:
##        print(birthdays[name] + ' is the birthday of ' + name)
##    else:
##        print('I do not have birthday info for ' + name)
##        print('What is their birthday?')
##        bday = input()
##        birthdays[name] = bday
##        print('Birthday database updated.')
##

##spam = {'color':'red','age':42}
##
##for v in spam.values():
##    print(v)
##
##for k in spam.keys():
##    print(k)
##
##for i in spam.items(): #items returns the key and value as a list
##    print(i)
##
##for k,v in spam.items():
##    print('key: ' + k +'. value: ' + str(v))
##
####checking whether a key or value exists in a dictionary
##
##spam = {'name':'Zophie','age':7}
##print(('name' in spam.keys())) # returns True

##picnicItems = {'apples':5,'cups':2}
##print('I am bringing ' + str(picnicItems.get('apples',0)) + ' apples.')
##print('I am bringing ' + str(picnicItems.get('eggs',0)) + ' eggs.')
##
## without the get method, this would return an error

##spam = {'name':'Pookie','age':5}
####if 'color' not in spam:
####    spam['color'] = 'black'
##spam.setdefault('color','black')
##
##for i in spam.items():
##    print(i)
##

import pprint #pretty print

message = 'It was a bright cold day in April, and the clocks were striking thirteen'
count = {}

for character in message:
    count.setdefault(character, 0)
    count[character] = count[character] + 1

pprint.pprint(count) #pprint seems to be just for dictionaries?


import pandas as pd
import sys
import csv

# Takes the merged.csv file and returns a cleaned, processed csv

CSV_dir = sys.argv[1] + "csv_data/"
DICT = "swap_dict_dists.csv"

# read dataset
df = pd.read_csv(CSV_dir+"preeditdist_processed.csv")
df = df[(~df.supplier.isna()) & (~df.buyer.isna())]
# drop the first column (index column)
df.drop(df.columns[0], axis=1, inplace=True)

replace_dict = {}

with open(DICT, "r") as csvfile:
    csvreader = csv.reader(csvfile)
    for row in csvreader:
        if row[0] == row[1]:
            continue
        replace_dict[row[0]] = row[1]

while len(df[df.buyer.isin(replace_dict.keys())]) > 0:
    print(f"Trying to replace {len(df[df.buyer.isin(replace_dict.keys())])} buyers")
    df.buyer.replace(replace_dict, inplace=True)

while len(df[df.supplier.isin(replace_dict.keys())]) > 0:
    print(f"Trying to replace {len(df[df.supplier.isin(replace_dict.keys())])} suppliers")
    df.supplier.replace(replace_dict, inplace=True)

df.to_csv(CSV_dir + "processed.csv")