"""Utilities for Processing Replies to Domain Read Requests."""
import base58
import base64
import hashlib
import json

from binascii import hexlify
from copy import deepcopy

from .utils import audit_path_length
from .constants import (
    ACCUM_FROM,
    ACCUM_TO,
    ALL_ATR_KEYS,
    LAST_SEQ_NO,
    VAL,
    VALUE,
    HASH,
    LAST_UPDATE_TIME,
    MARKER_CLAIM_DEF,
    MARKER_SCHEMA,
    MARKER_ATTR,
    MARKER_REVOC_DEF,
    MARKER_REVOC_REG_ENTRY,
    MARKER_REVOC_REG_ENTRY_ACCUM,
    GET_NYM,
    GET_ATTR,
    GET_CLAIM_DEF,
    GET_REVOC_REG_DEF,
    GET_REVOC_REG_ENTRY,
    GET_REVOC_REG_DELTA,
    GET_SCHEMA,
    NYM,
    ATTRIB,
    SCHEMA,
    CLAIM_DEF,
    REVOC_REG_DEF,
    REVOC_REG_ENTRY,
    DEST,
    RESULT,
    DATA,
    SEQ_NO,
    TXN_METADATA,
    TXN_TIME,
    TXN,
    NAME,
    VERSION,
    ATTR_NAMES,
    FROM,
    METADATA,
    REF,
    CRED_DEF_ID,
    REVOC_DEF_TYPE,
    REVOC_DEF_TYPE_ID,
    AUDIT_PATH,
    ROOT_HASH,
    STATE_PROOF,
    STATE_PROOF_FROM,
    PROOF_NODES,
    TAG,
)

def _extract_attr_typed_value(txn_data):
    """Check for 'raw', 'enc', 'hash' in ATTR & GET_ATTR, return it's name and value."""
    existing_keys = [key for key in ALL_ATR_KEYS if key in txn_data]
    if len(existing_keys) == 0:
        raise ValueError(
            "ATTR should have one of the following fields: {}".format(ALL_ATR_KEYS)
        )
    if len(existing_keys) > 1:
        raise ValueError(
            "ATTR should have only one of the following fields: {}".format(ALL_ATR_KEYS)
        )
    existing_key = existing_keys[0]
    return existing_key, txn_data[existing_key]

def parse_attr_txn(txn_data):
    """Process txn_data and parse attr_txn based on attr_type."""
    attr_type, attr = _extract_attr_typed_value(txn_data)
    if attr_type == "raw":
        data = json.loads(attr)
        re_raw = json.dumps(data)
        key, _ = data.popitem()
        return attr_type, key, re_raw
    if attr_type == "enc":
        return attr_type, attr, attr
    if attr_type == "hash":
        return attr_type, attr, None

def encode_state_value(value, seqNo, txnTime):
    """Return encoded state value."""
    return json.dumps({LAST_SEQ_NO: seqNo, LAST_UPDATE_TIME: txnTime, VAL: value})

def decode_state_value(encoded_value):
    """Return val, lsn, lut from encoded state value."""
    decoded = json.loads(encoded_value)
    value = decoded.get(VAL)
    last_seq_no = decoded.get(LAST_SEQ_NO)
    last_update_time = decoded.get(LAST_UPDATE_TIME)
    return value, last_seq_no, last_update_time

def hash_of(text) -> str:
    """Return 256 bit hexadecimal digest of text."""
    if not isinstance(text, (str, bytes)):
        text = json.dumps(text)
    if not isinstance(text, bytes):
        text = text.encode()
    return hashlib.sha256(text).hexdigest()

def make_state_path_for_attr(did, attr_name, attr_is_hash=False) -> bytes:
    """Return state_path for ATTR."""
    nameHash = (
        hashlib.sha256(attr_name.encode()).hexdigest()
        if not attr_is_hash
        else attr_name
    )
    return "{DID}:{MARKER}:{ATTR_NAME}".format(
        DID=did, MARKER=MARKER_ATTR, ATTR_NAME=nameHash
    ).encode()

def prepare_get_attr_for_state(reply):
    """Return value for state from GET_ATTR."""
    attr_type, attr_key = _extract_attr_typed_value(reply)
    data = reply.get(DATA)
    value_bytes = None
    if data:
        result = reply.copy()
        data = result.pop(DATA)
        result[attr_type] = data
        attr_type, _, value = parse_attr_txn(result)
        hashed_value = hash_of(value) if value else ""
        seq_no = result.get(SEQ_NO)
        txn_time = result.get(TXN_TIME)
        value_bytes = encode_state_value(hashed_value, seq_no, txn_time)
    return value_bytes

def prepare_attr_for_state(txn, path_only=False):
    """Return key, value pair for state from ATTR."""
    txn_data = txn.get(TXN, {}).get(DATA)
    nym = txn_data.get(DEST)
    attr_type, attr_key, value = parse_attr_txn(txn_data)
    path = make_state_path_for_attr(nym, attr_key, attr_type == HASH)
    if path_only:
        return path
    hashed_value = hash_of(value) if value else ""
    seq_no = txn[TXN_METADATA].get(SEQ_NO)
    txn_time = txn[TXN_METADATA].get(TXN_TIME)
    value_bytes = encode_state_value(hashed_value, seq_no, txn_time)
    return path, value_bytes.encode()

def make_state_path_for_nym(did) -> bytes:
    """Return state_path for NYM."""
    return hashlib.sha256(did.encode()).digest()

def prepare_nym_for_state(txn):
    """Return encoded state path from NYM."""
    txn_data = txn.get(TXN, {}).get(DATA)
    nym = txn_data.get(DEST)
    path = make_state_path_for_nym(nym)
    return hexlify(path).decode()

def prepare_get_nym_for_state(reply):
    """Return value for state from GET_NYM."""
    data = reply.get(DATA)
    value = None
    if data is not None:
        if isinstance(data, str):
            data = json.loads(data)
        data.pop(DEST, None)
        value = json.dumps(data)
    return value

def prepare_get_schema_for_state(reply):
    """Return value for state from GET_SCHEMA."""
    value_bytes = None
    attr_names = reply[DATA].get(ATTR_NAMES)
    if attr_names:
        data = {ATTR_NAMES: attr_names}
        seq_no = reply.get(SEQ_NO)
        txn_time = reply.get(TXN_TIME)
        value_bytes = encode_state_value(data, seq_no, txn_time)
    return value_bytes

def make_state_path_for_schema(authors_did, schema_name, schema_version) -> bytes:
    """Return state_path for SCHEMA."""
    return "{DID}:{MARKER}:{SCHEMA_NAME}:{SCHEMA_VERSION}".format(
        DID=authors_did,
        MARKER=MARKER_SCHEMA,
        SCHEMA_NAME=schema_name,
        SCHEMA_VERSION=schema_version,
    ).encode()

def prepare_schema_for_state(txn, path_only=False):
    """Return key-value pair for state from SCHEMA."""
    origin = txn[TXN].get(METADATA, {}).get(FROM)
    schema_name = txn[TXN][DATA][DATA].get(NAME)
    schema_version = txn[TXN][DATA][DATA].get(VERSION)
    value = {ATTR_NAMES: txn[TXN][DATA][DATA].get(ATTR_NAMES)}
    path = make_state_path_for_schema(origin, schema_name, schema_version)
    if path_only:
        return path
    seq_no = txn[TXN_METADATA].get(SEQ_NO)
    txn_time = txn[TXN_METADATA].get(TXN_TIME)
    value_bytes = encode_state_value(value, seq_no, txn_time)
    return path, value_bytes.encode()

def prepare_get_claim_def_for_state(reply):
    """Return value for state from GET_CLAIM_DEF."""
    schema_seq_no = reply.get(REF)
    if schema_seq_no is None:
        raise ValueError("ref field is absent, but it must contain schema seq no")
    value_bytes = None
    data = reply.get(DATA)
    if data is not None:
        seq_no = reply.get(SEQ_NO)
        txn_time = reply.get(TXN_TIME)
        value_bytes = encode_state_value(data, seq_no, txn_time)
    return value_bytes

def make_state_path_for_claim_def(authors_did, schema_seq_no, signature_type, tag):
    """Return state_path for CLAIM DEF."""
    return "{DID}:{MARKER}:{SIGNATURE_TYPE}:{SCHEMA_SEQ_NO}:{TAG}".format(
        DID=authors_did,
        MARKER=MARKER_CLAIM_DEF,
        SIGNATURE_TYPE=signature_type,
        SCHEMA_SEQ_NO=schema_seq_no,
        TAG=tag,
    ).encode()

def prepare_claim_def_for_state(txn, path_only=False):
    """Return key-value pair for state from CLAIM_DEF."""
    origin = txn[TXN].get(METADATA, {}).get(FROM)
    schema_seq_no = txn[TXN][DATA].get(REF)
    if schema_seq_no is None:
        raise ValueError("ref field is absent, but it must contain schema seq no")
    data = txn[TXN][DATA].get(DATA)
    if data is None:
        raise ValueError("data field is absent, but it must contain components of keys")
    signature_type = txn[TXN][DATA].get("signature_type", "CL")
    tag = txn[TXN][DATA].get(TAG, "tag")
    path = make_state_path_for_claim_def(origin, schema_seq_no, signature_type, tag)
    if path_only:
        return path
    seq_no = txn[TXN_METADATA].get(SEQ_NO)
    txn_time = txn[TXN_METADATA].get(TXN_TIME)
    value_bytes = encode_state_value(data, seq_no, txn_time)
    return path, value_bytes.encode()

def prepare_get_revoc_def_for_state(reply):
    """Return value for state from GET_REVOC_DEF."""
    seq_no = reply.get(SEQ_NO)
    txn_time = reply.get(TXN_TIME)
    value_bytes = encode_state_value(reply.get(DATA), seq_no, txn_time)
    return value_bytes

def make_state_path_for_revoc_def(
    authors_did, cred_def_id, revoc_def_type, revoc_def_tag
) -> bytes:
    """Return state_path for REVOC_DEF."""
    return "{DID}:{MARKER}:{CRED_DEF_ID}:{REVOC_DEF_TYPE}:{REVOC_DEF_TAG}".format(
        DID=authors_did,
        MARKER=MARKER_REVOC_DEF,
        CRED_DEF_ID=cred_def_id,
        REVOC_DEF_TYPE=revoc_def_type,
        REVOC_DEF_TAG=revoc_def_tag,
    ).encode()

def prepare_revoc_def_for_state(txn, path_only=False):
    """Return key-value pair for state from REVOC_DEF."""
    author_did = txn[TXN].get(METADATA, {}).get(FROM, None)
    txn_data = txn[TXN].get(DATA)
    cred_def_id = txn_data.get(CRED_DEF_ID)
    revoc_def_type = txn_data.get(REVOC_DEF_TYPE)
    revoc_def_tag = txn_data.get(TAG)
    path = make_state_path_for_revoc_def(
        author_did, cred_def_id, revoc_def_type, revoc_def_tag
    )
    if path_only:
        return path
    seq_no = txn[TXN_METADATA].get(SEQ_NO)
    txn_time = txn[TXN_METADATA].get(TXN_TIME)
    value_bytes = encode_state_value(txn_data, seq_no, txn_time)
    return path, value_bytes.encode()

def prepare_get_revoc_reg_entry_for_state(reply):
    """Return value for state from GET_REVOC_REG_ENTRY."""
    if RESULT in reply.keys():
        result = reply.get(RESULT)
    else:
        result = reply
    seq_no = result.get(SEQ_NO)
    txn_time = result.get(TXN_TIME)
    value_bytes = encode_state_value(result.get(DATA), seq_no, txn_time)
    return value_bytes

def make_state_path_for_revoc_reg_entry(revoc_reg_def_id) -> bytes:
    """Return state_path for REVOC_REG_ENTRY."""
    return "{MARKER}:{REVOC_REG_DEF_ID}".format(
        MARKER=MARKER_REVOC_REG_ENTRY, REVOC_REG_DEF_ID=revoc_reg_def_id
    ).encode()

def prepare_get_revoc_reg_delta_for_state(reply):
    """Return value for state from GET_REVOC_REG_DELTA."""
    if STATE_PROOF_FROM in reply[DATA]:
        accum_to_seq_no = reply[DATA][VALUE].get(ACCUM_TO, {}).get(SEQ_NO)
        accum_to_txn_time = reply[DATA][VALUE].get(ACCUM_TO, {}).get(TXN_TIME)
        accum_from_seq_no = reply[DATA][VALUE].get(ACCUM_FROM, {}).get(SEQ_NO)
        accum_from_txn_time = reply[DATA][VALUE].get(ACCUM_FROM, {}).get(TXN_TIME)
        return (
            encode_state_value(
                reply[DATA][VALUE].get(ACCUM_TO),
                accum_to_seq_no,
                accum_to_txn_time,
            ),
            encode_state_value(
                reply[DATA][VALUE].get(ACCUM_FROM),
                accum_from_seq_no,
                accum_from_txn_time,
            ),
        )
    else:
        seq_no = reply.get(SEQ_NO)
        txn_time = reply.get(TXN_TIME)
        return encode_state_value(reply[DATA][VALUE].get(ACCUM_TO), seq_no, txn_time)

def prepare_revoc_reg_entry_for_state(txn, path_only=False):
    """Return key-value pair for state from REVOC_REG_ENTRY."""
    txn_data = txn[TXN].get(DATA)
    revoc_reg_def_id = txn_data.get(REVOC_DEF_TYPE_ID)
    path = make_state_path_for_revoc_reg_entry(revoc_reg_def_id=revoc_reg_def_id)
    if path_only:
        return path
    seq_no = txn[TXN_METADATA].get(SEQ_NO)
    txn_time = txn[TXN_METADATA].get(TXN_TIME)
    txn_data = deepcopy(txn_data)
    txn_data[SEQ_NO] = seq_no
    txn_data[TXN_TIME] = txn_time
    value_bytes = encode_state_value(txn_data, seq_no, txn_time)
    return path, value_bytes.encode()

def prepare_get_revoc_reg_entry_accum_for_state(reply):
    """Return value for state from GET_REVOC_REG_ENTRY_ACCUM."""
    seq_no = reply.get(SEQ_NO)
    txn_time = reply.get(TXN_TIME)
    value_bytes = encode_state_value(reply.get(DATA), seq_no, txn_time)
    return value_bytes

def make_state_path_for_revoc_reg_entry_accum(revoc_reg_def_id) -> bytes:
    """Return state_path for REVOC_REG_ENTRY_ACCUM."""
    return "{MARKER}:{REVOC_REG_DEF_ID}".format(
        MARKER=MARKER_REVOC_REG_ENTRY_ACCUM, REVOC_REG_DEF_ID=revoc_reg_def_id
    ).encode()

def prepare_revoc_reg_entry_accum_for_state(txn):
    """Return key-value pair for state from REVOC_REG_ENTRY_ACCUM."""
    if RESULT in txn.keys():
        result = txn.get(RESULT)
    else:
        result = txn
    txn_data = result[TXN].get(DATA)
    revoc_reg_def_id = txn_data.get(REVOC_DEF_TYPE_ID)
    seq_no = result[TXN_METADATA].get(SEQ_NO)
    txn_time = result[TXN_METADATA].get(TXN_TIME)
    path = make_state_path_for_revoc_reg_entry_accum(revoc_reg_def_id=revoc_reg_def_id)
    txn_data = deepcopy(txn_data)
    txn_data[SEQ_NO] = seq_no
    txn_data[TXN_TIME] = txn_time
    value_bytes = encode_state_value(txn_data, seq_no, txn_time)
    return path, value_bytes.encode()

def extract_params_write_request(data):
    """Return tree_size, leaf_index, audit_path, expected_root_hash from reply."""
    if RESULT in data.keys():
        data = data.get(RESULT)
    tree_size = data[TXN_METADATA].get(SEQ_NO)
    leaf_index = tree_size - 1
    audit_path = data.get(AUDIT_PATH)
    audit_path = audit_path[:]
    decoded_audit_path = [
        base58.b58decode(hash_str.encode("utf-8")) for hash_str in audit_path
    ]
    expected_root_hash = base58.b58decode(data.get(ROOT_HASH).encode("utf-8"))
    if len(decoded_audit_path) != audit_path_length(leaf_index, tree_size):
        raise Exception("auditPath length does not match with given seqNo")
    return tree_size, leaf_index, decoded_audit_path, expected_root_hash

def get_proof_nodes(reply):
    """Return proof_nodes from reply."""
    if RESULT in reply.keys():
        reply = reply.get(RESULT)
    if reply.get("type") == GET_REVOC_REG_DELTA and STATE_PROOF_FROM in reply.get(DATA):
        proof_nodes_accum_to = reply[STATE_PROOF].get(PROOF_NODES)
        proof_nodes_accum_from = reply[DATA].get(STATE_PROOF_FROM, {}).get(PROOF_NODES)
        return base64.b64decode(proof_nodes_accum_to), base64.b64decode(
            proof_nodes_accum_from
        )
    else:
        b64_encoded_nodes = reply[STATE_PROOF].get(PROOF_NODES)
        return base64.b64decode(b64_encoded_nodes)

def prepare_for_state_read(reply):
    """Return state value from read requests reply."""
    if RESULT in reply.keys():
        reply = reply.get(RESULT)
    request_type = reply.get("type")
    if request_type == GET_ATTR:
        return prepare_get_attr_for_state(reply=reply)
    if request_type == GET_NYM:
        return prepare_get_nym_for_state(reply=reply)
    if request_type == GET_SCHEMA:
        return prepare_get_schema_for_state(reply=reply)
    if request_type == GET_CLAIM_DEF:
        return prepare_get_claim_def_for_state(reply=reply)
    if request_type == GET_REVOC_REG_DEF:
        return prepare_get_revoc_def_for_state(reply=reply)
    if request_type == GET_REVOC_REG_ENTRY:
        return prepare_get_revoc_reg_entry_accum_for_state(reply=reply)
    if request_type == GET_REVOC_REG_DELTA:
        if "issued" in reply[DATA].get("value"):
            return prepare_get_revoc_reg_delta_for_state(reply=reply)
        else:
            return prepare_get_revoc_reg_entry_accum_for_state(reply=reply)
    raise ValueError(
        "Cannot make state value for request of type {}".format(request_type)
    )

def prepare_for_state_write(reply):
    """Return state key, value pair from write requests reply."""
    if RESULT in reply.keys():
        reply = reply.get(RESULT)
    request_type = reply[TXN].get("type")
    if request_type == NYM:
        return prepare_nym_for_state(txn=reply)
    if request_type == ATTRIB:
        return prepare_attr_for_state(txn=reply)
    if request_type == SCHEMA:
        return prepare_schema_for_state(txn=reply)
    if request_type == CLAIM_DEF:
        return prepare_claim_def_for_state(txn=reply)
    if request_type == REVOC_REG_DEF:
        return prepare_revoc_def_for_state(txn=reply)
    if request_type == REVOC_REG_ENTRY:
        return prepare_revoc_reg_entry_for_state(txn=reply)
    raise ValueError(
        "Cannot make state key-value pair for request of type {}".format(request_type)
    )


import heapq
import sys

v,e = map(int, input().split())

inf = 987654321

adjdic = dict()

for i in range(e):
    a,b,c = map(int, sys.stdin.readline().split())
    if a in adjdic.keys():
        if b in adjdic[a].keys():
            adjdic[a][b] = min(adjdic[a][b], c)
        else:
            adjdic[a][b] = c
    else:
        adjdic[a] = {b:c}
    if b in adjdic.keys():
        if a in adjdic[a].keys():
            adjdic[b][a] = min(adjdic[b][a], c)
        else:
            adjdic[b][a] = c
    else:
        adjdic[b] = {a:c}

oil1, oil2 = map(int,input().split())

def finddest(src,dest):
    dijkmat = [inf for _ in range(v+1)]
    dijkmat[src] = 0

    possible = [] # weight, source, target vertex
    visited = set()
    visited.add(src)
    if src in adjdic.keys():
        for i in adjdic[src].keys():
            dijkmat[i] = adjdic[src][i]
            heapq.heappush(possible, (adjdic[src][i], src, i))

    while(len(possible) > 0 and len(visited)!=v):
        todo = heapq.heappop(possible)
        if not todo[2] in adjdic.keys():
            continue
        for i in adjdic[todo[2]].keys():
            if dijkmat[i] > adjdic[todo[2]][i]+dijkmat[todo[2]]:
                dijkmat[i] = adjdic[todo[2]][i]+dijkmat[todo[2]]
            if not todo[2] in visited:
                heapq.heappush(possible, (adjdic[todo[2]][i]+dijkmat[todo[2]], todo[2], i))
        visited.add(todo[2])

    return dijkmat[dest]

way1 = finddest(1, oil1)
way2 = finddest(oil1, oil2)
way3 = finddest(oil2, v)
way4 = finddest(1, oil2)
way5 = finddest(oil1, v)

if way1+way3 > way4+way5:
    if way4+way5+way2 >= inf:
        print(-1)
    else:
        print(way4+way5+way2)
else:
    if way1+way3+way2 >= inf:
        print(-1)
    else:
        print(way1+way3+way2)


import logging
from collections import OrderedDict
from PyQt5 import QtWidgets
from PyQt5.QtCore import Qt, pyqtSignal, pyqtSlot, QSize
from PyQt5.QtGui import QBrush, QColor, QIcon, QCursor, QFont
from PyQt5.QtWidgets import QDialogButtonBox, QStyle
from .. import sqlitedb
from ..tools import fuzzyMatchPrepare, fuzzyMatch, dfsCC, getHLine, parseAuthors
from .threadrun_dialog import Master
from .search_res_frame import AdjustableTextEditWithFold

LOGGER=logging.getLogger(__name__)

class CheckDuplicateFrame(QtWidgets.QScrollArea):

    del_doc_from_folder_signal=pyqtSignal(list, # docids
            str, #foldername
            str, #folderid
            bool #reload_talbe
            )
    del_doc_from_lib_signal=pyqtSignal(list, # docids
            bool, #reload_table
            bool # ask
            )

    def __init__(self,settings,parent=None):
        '''
        Args:
            settings (QSettings): application settings. See _MainWindow.py
            parent (QWidget): parent widget.
        '''

        super(CheckDuplicateFrame,self).__init__(parent=parent)

        self.settings=settings
        self.parent=parent
        self.min_score=self.settings.value('duplicate_min_score',type=int)

        frame=QtWidgets.QWidget()
        self.setWidgetResizable(True)
        self.setWidget(frame)
        va=QtWidgets.QVBoxLayout(self)

        #----------------Create clear frame----------------
        va.addWidget(self.createClearDuplicateFrame())

        #----------------Create treewidget----------------
        self.tree=QtWidgets.QTreeWidget(self)
        self.tree.setColumnCount(7)
        self.tree.setColumnHidden(6,True)

        self.tree.setHeaderLabels(['Group', 'Authors', 'Title',
            'Publication', 'Year', 'Similarity','id'])
        self.tree.setColumnWidth(0, 55)
        self.tree.setColumnWidth(1, 250)
        self.tree.setColumnWidth(2, 300)
        self.tree.setColumnWidth(3, 150)
        self.tree.setColumnWidth(4, 50)
        self.tree.setColumnWidth(5, 20)
        self.tree.setColumnWidth(6, 0)
        self.tree.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        self.tree.setHorizontalScrollMode(QtWidgets.QAbstractItemView.ScrollPerPixel)
        self.tree.setVerticalScrollMode(QtWidgets.QAbstractItemView.ScrollPerPixel)
        self.tree.header().setStretchLastSection(False)
        self.tree.header().setSectionResizeMode(
                QtWidgets.QHeaderView.Interactive)
        self.tree.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
        self.tree.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
        self.tree.setDragDropMode(QtWidgets.QAbstractItemView.NoDragDrop)
        self.tree.setContextMenuPolicy(Qt.CustomContextMenu)
        self.tree.customContextMenuRequested.connect(self.docTreeMenu)

        #----------------Create merge frame----------------
        self.merge_frame=MergeFrame(self.settings, self.tree)
        self.merge_frame.setVisible(False)
        self.merge_frame.change_to_table_sig.connect(self.changeView)

        self.no_dup_label=QtWidgets.QLabel('No duplicates found.')
        self.no_dup_label.setVisible(False)
        va.addWidget(self.no_dup_label)
        va.addWidget(self.tree)
        va.addWidget(self.merge_frame)

        frame.setLayout(va)

    def createClearDuplicateFrame(self):

        frame=QtWidgets.QFrame()
        frame.setStyleSheet('background: rgb(235,225,190)')
        ha=QtWidgets.QHBoxLayout()
        self.current_view='table'  # 'table' | 'merge'

        # del button
        self.del_duplicate_button=QtWidgets.QToolButton(self)
        self.del_duplicate_button.setText('Delete Selected')
        self.del_duplicate_button.clicked.connect(self.delDocs)

        # change to merge/table view button
        self.change_view_button=QtWidgets.QToolButton(self)
        self.change_view_button.setText('Change to Merge View')
        self.change_view_button.clicked.connect(self.changeView)

        # clear button
        self.clear_duplicate_button=QtWidgets.QToolButton(self)
        self.clear_duplicate_button.setText('Exit')

        self.clear_duplicate_label=QtWidgets.QLabel()
        ha.addWidget(self.clear_duplicate_label)
        tip_label=QtWidgets.QLabel()
        tip_icon=QIcon.fromTheme('help-about',
            self.style().standardIcon(QStyle.SP_MessageBoxInformation)).pixmap(
                    QSize(16,16))
        tip_label.setPixmap(tip_icon)
        tip_label.setToolTip('''Change "Mininimum Similary Score" in "Preferences" to change the filtering of matching results.''')

        ha.addWidget(tip_label)
        ha.addWidget(self.del_duplicate_button)
        ha.addWidget(self.change_view_button)
        ha.addWidget(self.clear_duplicate_button)

        frame.setLayout(ha)

        return frame

    @pyqtSlot()
    def changeView(self):
        '''Change between the table/merge view'''

        current=self.tree.currentItem()

        #---------------Change to merge view---------------
        if self.current_view=='table' and current:
            self.change_view_button.setText('Change to Table View')
            self.current_view='merge'
            self.del_duplicate_button.setVisible(False)
            self.tree.setVisible(False)
            self.merge_frame.setVisible(True)

            # current is a child row
            if current.data(0,0)=='':
                gid=current.parent().data(0,0)
            else:
                gid=current.data(0,0)

            LOGGER.debug('current_item gid = %s' %current.data(0,0))

            self.merge_frame.createMergeForGroup(gid, self.group_dict,
                    self.meta_dict, self.folder_dict, self.current_folder)

        #---------------Change to table view---------------
        # NOTE: I said 'table' but it's really a QTreeWidget
        elif self.current_view=='merge':
            self.change_view_button.setText('Change to Merge View')
            self.current_view='table'
            self.del_duplicate_button.setVisible(True)
            self.tree.setVisible(True)
            self.merge_frame.setVisible(False)

        LOGGER.info('View changed to %s' %self.current_view)

        return

    def checkDuplicates(self, meta_dict, folder_dict, current_folder, docids1,
            docid2=None):
        """Start the duplicate checking among documents

        Args:
            meta_dict (dict): meta data of all documents. keys: docid,
                              values: DocMeta dict.
            folder_dict (dict): folder structure info. keys: folder id in str,
                values: (foldername, parentid) tuple.
            current_folder (tuple): (foldername_in_str, folder_id_in_str).
            docids1 (list): ids of docs in group 1.

        Kwargs:
            docid2 (int, list or None): if int or list, ids of docs in group 2,
                then duplicate checking is done among all pairs across group 1,
                2. If None, checking is done among all pairs in group 1.
        """

        self.tree.clear()
        self.setVisible(True)
        if self.current_view=='merge':
            self.changeView()

        self.meta_dict=meta_dict
        self.folder_dict=folder_dict
        self.current_folder=current_folder # (name, id)
        self.docids1=docids1
        self.docids1.sort()
        self.docid2=docid2

        #---------Disable merging if inside trash---------
        trashed_folder_ids=sqlitedb.getTrashedFolders(self.folder_dict)
        if current_folder[1] in trashed_folder_ids:
            self.change_view_button.setEnabled(False)
        else:
            self.change_view_button.setEnabled(True)

        self.scores_dict={}

        self.master1=Master(self.prepareJoblist, [(0,self.docids1,self.docid2)],
                1, self.parent.progressbar,
                'busy', self.parent.status_bar, 'Preparing job list...')
        self.master1.all_done_signal.connect(self.jobListReady)
        self.master1.run()

        return

    def prepareJoblist(self, jobid, docids1, docid2):
        '''Prepare fuzzy matching jobs

        Args:
            jobid (int): dummy job id.
            docids1 (list): ids of docs in group 1.
            docid2 (int, list or None): ids of docs in group 2,

        Returns:
            rec (int): 0 for success.
            jobid (int): dummy job id.
            job_list (list): list of tuples, each providing the args for a
                             fuzzyMatch() call.
        '''

        n=len(docids1)
        job_list=[]
        cache_dict={}  # store strings for docs to avoid re-compute

        def getFromCache(cdict, key):
            if key in cdict:
                value=cdict[key]
            else:
                value=fuzzyMatchPrepare(key, self.meta_dict[key])
                cdict[key]=value
            return value

        #----------------Check among docds----------------
        if docid2 is None:
            jobid2=0
            for ii in range(n):
                docii=docids1[ii]
                _, authorsii, titleii, jyii=getFromCache(cache_dict, docii)
                for jj in range(n):
                    docjj=docids1[jj]
                    if ii>=jj:
                        self.scores_dict[(docii, docjj)]=0
                    else:
                        _, authorsjj, titlejj, jyjj=getFromCache(cache_dict,
                                docjj)

                        # shortcut: skip if author string len diff >= 50%
                        if abs(len(authorsii)-len(authorsjj))>=\
                                max(len(authorsii), len(authorsjj))//2:
                            self.scores_dict[(docii, docjj)]=0
                            continue

                        # shortcut: skip if title string len diff >= 50%
                        if abs(len(titleii)-len(titlejj))>=\
                                max(len(titleii), len(titlejj))//2:
                            self.scores_dict[(docii, docjj)]=0
                            continue

                        job_list.append((jobid2,
                            getFromCache(cache_dict, docii),
                            getFromCache(cache_dict, docjj),
                            self.min_score))
                        jobid2+=1

        #-----------------nxm compare-----------------
        else:
            if not isinstance(docid2, (tuple,list)):
                # docid2 is a single doc
                docid2=[docid2,]

            m=len(docid2)
            jobid2=0

            for ii in range(m):
                docii=docid2[ii]
                _, authorsii, titleii, jyii=getFromCache(cache_dict, docii)
                for jj in range(n):
                    docjj=docids1[jj]
                    if docii!=docjj:
                        _, authorsjj, titlejj, jyjj=getFromCache(cache_dict,
                                docjj)

                        # shortcut: skip if author string len diff >= 50%
                        if abs(len(authorsii)-len(authorsjj))>=\
                                max(len(authorsii), len(authorsjj))//2:
                            self.scores_dict[(docii, docjj)]=0
                            continue

                        # shortcut: skip if title string len diff >= 50%
                        if abs(len(titleii)-len(titlejj))>=\
                                max(len(titleii), len(titlejj))//2:
                            self.scores_dict[(docii, docjj)]=0
                            continue

                        job_list.append((jobid2,
                            getFromCache(cache_dict, docii),
                            getFromCache(cache_dict, docjj),
                            self.min_score))
                        jobid2+=1

        return 0,jobid,job_list

    @pyqtSlot()
    def jobListReady(self):
        '''After getting all jobs, launch threads and process jobs
        '''

        rec,jobid,job_list=self.master1.results[0]
        LOGGER.debug('rec from job list prepare = %s' %rec)

        if rec==0 and len(job_list)>0:
            self.parent.progressbar.setMaximum(0)
            self.parent.progressbar.setVisible(True)
            # make a separate master for fuzzy matching. This separation of
            # joblist-preparing and fuzzy matching is for easy aborting.
            # For large data size, it might take a few seconds to get the job
            # list prepared, therefore 2 threaded calls.
            self.master2=Master(fuzzyMatch,job_list,1,self.parent.progressbar,
                    'classic',self.parent.status_bar,'Computing Fuzzy Matching...')
            self.master2.all_done_signal.connect(self.collectResults)
            self.clear_duplicate_button.clicked.connect(self.master2.abortJobs)
            self.master2.run()

        return

    @pyqtSlot()
    def collectResults(self):
        '''Collect matching results and send results to GUI'''

        new=self.master2.results
        for recii,jobidii,resii in new:
            if recii==0:
                kii,vii=resii
                self.scores_dict[kii]=vii

        LOGGER.info('Duplicate search results collected.')
        self.addResults()

        return

    def createEntry(self, docid, gid, score):
        '''Create a QTreeWidgetItem for a doc

        Args:
            docid (int): doc id.
            gid (int): group id.
            score (int): similarity score.

        Return: item (QTreeWidgetItem): QTreeWidgetItem to add to the tree
        '''

        meta=self.meta_dict[docid]
        item=QtWidgets.QTreeWidgetItem([
            gid,
            ', '.join(meta['authors_l']),
            meta['title'],
            meta['publication'],
            str(meta['year']),
            score,
            str(docid)
            ])

        return item

    def addResults(self):
        '''Add matching results to treewidget'''

        edges=[kk for kk,vv in self.scores_dict.items() if vv>=self.min_score]
        hi_color=self.settings.value('display/folder/highlight_color_br',
                QBrush)

        # if no duplicates, return
        if len(edges)==0:
            self.no_dup_label.setVisible(True)
            LOGGER.info('No duplicate found.')
            return

        self.no_dup_label.setVisible(False)

        #--------------------Get groups--------------------
        groups={}  # key: gid, value: list of doc ids in same group

        if self.docid2 is None:
            # get connected components
            comps=dfsCC(edges)
            LOGGER.debug('connected components = %s' %comps)
            for ii,cii in enumerate(comps):
                cii.sort()
                groups[ii]=cii
        else:
            if not isinstance(self.docid2, (tuple,list)):
                # docid2 is a single doc
                docid2=[self.docid2,]
            else:
                docid2=self.docid2

            for ii,docii in enumerate(docid2):
                docs=[kk[1] for kk in edges if kk[0]==docii]
                if len(docs)>0:
                    groups[ii]=[docii]+docs # don't forget docii itself

        #--------------------Add header rows--------------------
        # sort by group length
        headers=sorted(groups.keys(), key=lambda x: len(groups[x]),
                reverse=True)

        self.group_dict=OrderedDict()
        # key: groupid, value: dict: {'header': header QTreeWidgetItem,
        #                             'members': list of doc ids in group}
        # this dict is later used to reconstruct tree after a merging.

        for ii,gii in enumerate(headers):

            members=groups[gii]
            docii=members[0]
            others=members[1:]
            newgid=ii+1
            itemii=self.createEntry(docii, str(newgid), '')

            # highlight group header
            for jj in range(self.tree.columnCount()):
                itemii.setBackground(jj, hi_color)

            self.tree.addTopLevelItem(itemii)

            #------------Create table view members------------
            # sort group members by scores
            scores=[]
            for djj in others:
                if (docii, djj) in edges:
                    sii=self.scores_dict[(docii, djj)]
                else:
                    sii=self.scores_dict[(djj, docii)]
                scores.append(sii)

            others=[x for _,x in sorted(zip(scores,others), reverse=True)]
            scores=sorted(scores,reverse=True)

            #----------------Add to group_dict----------------
            members=[docii]+others
            self.group_dict[newgid]={'header': itemii, 'members' : members}

            #-------------------Add to tree-------------------
            for jj,docjj in enumerate(members[1:]):
                # connection lost along the path between 2 vertices
                sjj=str(scores[jj])
                #if sjj=='0':
                    #sjj='N/A'
                itemjj=self.createEntry(docjj, '', sjj)
                itemii.addChild(itemjj)

        self.tree.setStyleSheet('''
        QTreeWidget::item:has-children { border-left: 1px solid black;
        background-color: %s;}
        ''' %hi_color.color().name())

        self.tree.expandAll()
        LOGGER.info('Duplicate search results added.')

        return

    def docTreeMenu(self,pos):
        '''Right click menu in the document tree'''

        menu=QtWidgets.QMenu()

        foldername,folderid=self.current_folder
        if folderid=='-1':
            menu.addAction('Delete From &Library')
        else:
            menu.addAction('Delete From Current &Folder')

        menu.addAction('&Remove From Duplicate Group')

        action=menu.exec_(QCursor.pos())

        if action:
            action_text=action.text().replace('&','')
            if action_text in ['Delete From Current Folder',
                    'Delete From Library']:
                self.delDocs()
            elif action_text=='Remove From Duplicate Group':
                self.removeFromGroup()

        return

    @pyqtSlot()
    def delDocs(self):

        LOGGER.debug('current_folder = %s' %str(self.current_folder))
        foldername,folderid=self.current_folder
        sel_rows=self.tree.selectedItems()

        if len(sel_rows)>0:

            docids=[int(ii.data(6,0)) for ii in sel_rows]
            LOGGER.debug('Selected docids = %s.' %docids)

            if folderid=='-1':
                LOGGER.info('Emit signal for doc deletion in All')
                self.del_doc_from_lib_signal.emit(docids,False,True)
            else:
                LOGGER.info('Emit signal for doc deletion in folder %s' %foldername)
                self.del_doc_from_folder_signal.emit(docids, foldername,
                        folderid, False)

            for itemii in sel_rows:
                self.tree.invisibleRootItem().removeChild(itemii)

        return

    def removeFromGroup(self):

        sel_rows=self.tree.selectedItems()
        hi_color=self.settings.value('display/folder/highlight_color_br',
                QBrush)

        #-------------------Remove items-------------------
        # while is better than for when dealing with item iter
        while len(sel_rows)>0:
            itemii=sel_rows.pop()
            docid=int(itemii.data(6,0))

            #------------Remove a child from group------------
            if itemii.data(0,0)=='':
                gid=int(itemii.parent().data(0,0))
                itemii.parent().removeChild(itemii)

                if gid in self.group_dict:
                    members=self.group_dict[gid]['members']
                    if docid in members:
                        members.remove(docid)
                    self.group_dict[gid]['members']=members

            #-------------Remove header from group-------------
            else:
                gid=int(itemii.data(0,0))
                members=self.group_dict[gid]['members']
                # single header, del
                if len(members)==1:
                    self.tree.invisibleRootItem().removeChild(itemii)
                    del self.group_dict[gid]
                else:
                    idx=self.tree.indexOfTopLevelItem(itemii)
                    children=itemii.takeChildren()
                    itemii=self.tree.takeTopLevelItem(idx)

                    if len(children)>=2:
                        if docid in members:
                            members.remove(docid)
                        self.group_dict[gid]['members']=members

                        # promote the next child to header
                        newheader=children[0]
                        newheader.setData(0,0,str(gid))
                        self.tree.insertTopLevelItem(idx, newheader)

                        # highlight group header
                        for jj in range(self.tree.columnCount()):
                            newheader.setBackground(jj, hi_color)

                        # add new children
                        for cjj in children[1:]:
                            newheader.addChild(cjj)

                        newheader.setExpanded(True)
                    else:
                        del self.group_dict[gid]

        #-------------Remove singleton headers-------------
        idx=0
        while idx<self.tree.topLevelItemCount():
            itemii=self.tree.topLevelItem(idx)
            if itemii.childCount()==0:
                self.tree.takeTopLevelItem(idx)
                gid=itemii.data(0,0)
                del self.group_dict[int(gid)]
            else:
                idx+=1

        LOGGER.debug('groups after deletion: %s' %self.group_dict.keys())

        return

class MergeFrame(QtWidgets.QScrollArea):

    # add new doc after merging. Connect to _MainFrameDataSlots.addDocFromDuplicateMerge()
    add_new_doc_sig=pyqtSignal(sqlitedb.DocMeta)

    # del docs after merging. Connect to _MainFrameDocTableSlots.delDoc()
    del_doc_sig=pyqtSignal(list, bool, bool)  # docids, reload_table, ask

    # change to table view after merging. Connect to CheckDuplicateFrame.changeView()
    change_to_table_sig=pyqtSignal()

    def __init__(self, settings, tree, parent=None):
        '''Interface for resolving conflicts in duplicate merging

        Args:
            settings (QSettings): application settings. See _MainWindow.py
            parent (QWidget): parent widget.
            tree (QTreeWidget): treewidget created in CheckDuplicateFrame.
        '''

        super(self.__class__, self).__init__(parent=parent)

        self.settings=settings
        self.tree=tree

        frame=QtWidgets.QWidget()
        frame.setStyleSheet('background-color:white')
        self.setWidgetResizable(True)
        self.setWidget(frame)
        va=QtWidgets.QVBoxLayout(self)
        #va.setContentsMargins(0,0,0,0)

        self.label=QtWidgets.QLabel('Resolve Conflicts')
        self.label.setStyleSheet('font: bold 14px;')

        # toggle all fields button
        self.toggle_all_button=QtWidgets.QToolButton(self)
        self.toggle_all_button.setText('Show All Fields')
        self.toggle_all_button.clicked.connect(self.toggleAllFields)
        self.show_all=False

        ha=QtWidgets.QHBoxLayout()
        ha.addWidget(self.label)
        ha.addWidget(self.toggle_all_button, 1, Qt.AlignRight)
        va.addLayout(ha)
        va.addWidget(getHLine())

        # grid layout, will be filled in createMergeForGroup
        self.merge_grid=QtWidgets.QGridLayout()
        va.addLayout(self.merge_grid)
        va.addStretch()

        # confirm button
        self.confirm_button=QtWidgets.QToolButton(self)
        self.confirm_button.setText('Confirm Merge')
        self.confirm_button.clicked.connect(self.confirmMerge)
        self.confirm_button.setStyleSheet('font: bold 12px;')

        # preview button
        #self.preview_button=QtWidgets.QToolButton(self)
        #self.preview_button.setText('Preview')
        #self.preview_button.clicked.connect(self.previewMerge)
        #self.preview_button.setStyleSheet('font: bold 12px;')

        ha=QtWidgets.QHBoxLayout()
        ha.addWidget(self.confirm_button, 0, Qt.AlignRight)
        #ha.addWidget(self.preview_button, 0, Qt.AlignRight)
        va.addLayout(ha)

        frame.setLayout(va)

        # complete field list not including 'folders_l'
        self.fields=[
                'title', 'issue', 'pages', 'publication', 'volume', 'year',
                'doi', 'abstract', 'arxivId', 'chapter', 'city', 'country',
                'edition', 'institution', 'isbn', 'issn', 'month', 'day',
                'publisher', 'series', 'pmid', 'keywords_l', 'files_l',
                'tags_l', 'urls_l', 'notes', 'authors_l'
                ]
        self.fields.sort()
        self.long_fields=['title', 'abstract', 'institution', #?
                'keywords_l', 'files_l', 'tags_l', 'urls_l', 'notes',
                'authors_l']

    def createMergeForGroup(self, gid, group_dict, meta_dict, folder_dict,
            current_folder):
        '''Create interface for merging a group

        Args:
            gid (int): group id.
            group_dict (dict): group info. key: groupid, value: dict:
                               {'header': header QTreeWidgetItem,
                                'members': list of doc ids in group}.
            meta_dict (dict): meta data of all documents. keys: docid,
                              values: DocMeta dict.
            folder_dict( dict): folder structure info. keys: folder id in str,
                                values: (foldername, parentid) tuple.
            current_folder (tuple): (folderid_in_str, foldername_in_str).
        '''

        def valueToStr(meta_dict, key):
            '''Convert meta dict field to str'''
            value=meta_dict[key]
            if value is None:
                return None
            if key.endswith('_l'):
                value='; '.join(value)
            else:
                value=str(value)
            return value

        #--------------------Store data--------------------
        gid=int(gid)  # gid was fetched from QTreeWidgetItem, get back to int
        self.gid=gid
        self.group_dict=group_dict
        self.meta_dict=meta_dict
        self.folder_dict=folder_dict
        self.current_folder=current_folder
        self.label.setText('Resolve Conflicts in Group %d' %gid)

        #-----------Collect meta data by fields-----------
        members=group_dict[gid]['members']
        meta_list=[meta_dict[ii] for ii in members]
        self.value_dict=OrderedDict() # key: field name (e.g. title), value:
        # non-empty list of unique values among members

        for kk in self.fields:
            valuekk=[]
            for dii in meta_list:
                vii=valueToStr(dii,kk)
                if vii != '' and vii is not None:
                    valuekk.append(vii)

            valuekk=list(set(valuekk))
            if len(valuekk)>0:
                self.value_dict[kk]=valuekk

        LOGGER.debug('gid = %s. members = %s' %(gid, members))

        #-------------------Collect folders-------------------
        folders=[]
        for dii in meta_list:
            folders.extend(map(tuple, dii['folders_l']))

        folders=list(set(folders))  # [(fid1, fname1), (fid2, fname2) ...]

        if len(folders)>0:  # it must be, unless in trash
            folders2=[]
            for fii in folders:
                folders2.append((str(fii[0]),  # folderid in str
                    sqlitedb.getFolderTree(self.folder_dict, str(fii[0]))[1])
                    # folder in tree e.g. folder1/sub1/subsub2
                    )
            self.value_dict['folders_l']=folders2

        LOGGER.debug('folders in group = %s' %folders2)
        self.addFieldRows()

        return

    def clearMergeGrid(self):
        '''Clear grid layout'''

        while self.merge_grid.count():
            child = self.merge_grid.takeAt(0)
            if child.widget():
                child.widget().deleteLater()

        return

    def addFieldRows(self):
        '''Add uniqute values in a field to the merge frame'''

        font=self.settings.value('display/fonts/doc_table',QFont)
        self.clearMergeGrid()
        grid=self.merge_grid
        crow=grid.rowCount()

        self.collect_dict={} # key: field name, value: buttongroup if field
        # has more than 1 value. str if only 1 value.
        self.button_le_dict={} # key: radiobutton/checkbox:
                               # value: textedit/lineedit/str

        #--------------------Add fields--------------------
        for fii, valuesii in self.value_dict.items():

            # put folders at the end
            if fii=='folders_l':
                continue

            LOGGER.debug('adding field %s. NO values = %d' %(fii, len(valuesii)))

            # if no conflict and not showing all fields
            if not self.show_all and len(valuesii)==1:
                # still add to collect_dict as it will be used for creating
                # new doc
                self.collect_dict[fii]=valuesii[0]
                continue

            rgroup=QtWidgets.QButtonGroup(self) # NOTE: has to add self
            rgroup.setExclusive(True)
            self.collect_dict[fii]=rgroup

            # add unique values
            for jj,vjj in enumerate(valuesii):

                radiojj=QtWidgets.QRadioButton()

                if jj==0:
                    labelii=QtWidgets.QLabel('%s: ' %fii.replace('_l','').capitalize())
                    # highlight fields with conflicts
                    if len(valuesii)>1:
                        labelii.setStyleSheet('font: bold; color: red;')
                    grid.addWidget(labelii,crow,0)
                    radiojj.setChecked(True)

                if fii in self.long_fields:
                    text_editjj=AdjustableTextEditWithFold(self)
                    text_editjj.unfoldText()
                    grid.addWidget(text_editjj.fold_button,crow,2)

                    # set tooltip texts
                    if fii=='authors_l':
                        text_editjj.label_enabled=True
                        text_editjj.tooltip_text='lastname, firstname; lastname, firstname;...'
                    elif fii=='tags_l':
                        text_editjj.label_enabled=True
                        text_editjj.tooltip_text='tag1; tag2; tag3 ...'
                    elif fii=='keywords_l':
                        text_editjj.label_enabled=True
                        text_editjj.tooltip_text='keyword1; keyword2; keyword3 ...'
                else:
                    text_editjj=QtWidgets.QLineEdit()
                text_editjj.setFont(font)
                text_editjj.setText(vjj)

                grid.addWidget(radiojj,crow,1)
                grid.addWidget(text_editjj,crow,3)
                rgroup.addButton(radiojj)
                self.button_le_dict[radiojj]=text_editjj

                crow+=1

        #------------Add folders as checkboxes------------
        folders=self.value_dict.get('folders_l')

        if folders:

            fii='folders_l'
            valuesii=folders

            rgroup=QtWidgets.QButtonGroup(self) # NOTE: has to add self
            rgroup.setExclusive(False)
            self.collect_dict[fii]=rgroup

            for jj,vjj in enumerate(valuesii):

                # vjj: (folderid_in_str, folder_tree_path_in_str)

                radiojj=QtWidgets.QCheckBox(self)
                radiojj.setChecked(True)
                # if only 1 folder, checking is within a folder, disable state
                # change
                if len(valuesii)==1:
                    radiojj.setEnabled(False)

                if jj==0:
                    labelii=QtWidgets.QLabel('Add to folder(s)')
                    if len(valuesii)>1:
                        labelii.setStyleSheet('font: bold; color: red;')
                    grid.addWidget(labelii,crow,0)

                text_editjj=QtWidgets.QLineEdit(self)
                text_editjj.setFont(font)
                text_editjj.setText(vjj[1])
                text_editjj.setReadOnly(True)

                grid.addWidget(radiojj,crow,1)
                grid.addWidget(text_editjj,crow,3)

                rgroup.addButton(radiojj)
                # NOTE: here value is folder id
                self.button_le_dict[radiojj]=vjj[0]

                crow+=1
                LOGGER.debug('added folder (%s, %s)' %(vjj[0], vjj[1]))

        return

    @pyqtSlot()
    def toggleAllFields(self):
        '''Switch between show all fields and show conflict only'''

        if self.show_all:
            self.toggle_all_button.setText('Show All Fields')
            self.show_all=False
        else:
            self.toggle_all_button.setText('Show Only Conflicts')
            self.show_all=True

        self.addFieldRows()

        return

    @pyqtSlot()
    def confirmMerge(self):
        '''Confirm a duplicate merge

        This will merge a group of duplicates into a unified document. A new
        doc is created from the fields picked by user, and previous docs in
        the group will be send to Trash.
        '''

        def parseToList(text):
            '''Text to list'''
            result=[]
            textlist=text.replace('\n',';').strip(';').split(';')
            for tii in textlist:
                tii=tii.strip()
                if len(tii)>0:
                    result.append(tii)
            return result

        members=self.group_dict[self.gid]['members']
        LOGGER.info('Merging group %s. Members = %s' %(self.gid, members))

        result_dict=sqlitedb.DocMeta()

        #---Get text from the checked textedit/lineedit---
        for fii, rgroupii in self.collect_dict.items():

            # special treatment
            if fii=='folders_l':
                continue

            # rgroupii is buttongroup: more than 1 value
            if isinstance(rgroupii, QtWidgets.QButtonGroup):
                checked=rgroupii.checkedButton()
                # get the corresponding textedit/lineedit
                textwidget=self.button_le_dict[checked]

                if isinstance(textwidget, QtWidgets.QTextEdit):
                    textii=self.button_le_dict[checked].toPlainText()
                elif isinstance(textwidget, QtWidgets.QLineEdit):
                    textii=self.button_le_dict[checked].text()

                LOGGER.debug('Conflict field = %s. Chosen value = %s'\
                        %(fii, textii))
            # no conflict, single value
            else:
                textii=rgroupii
                LOGGER.debug('Agreed field = %s. Chosen value = %s'\
                        %(fii, textii))

            if fii.endswith('_l'):
                values=parseToList(textii.strip())
                if fii=='authors_l':
                    firsts,lasts,_=parseAuthors(values)
                    result_dict['firstNames_l']=firsts
                    result_dict['lastName_l']=lasts
                else:
                    result_dict[fii]=values
            else:
                result_dict[fii]=textii.strip()

        #-------------------Get folders-------------------
        rgroup=self.collect_dict.get('folders_l')
        folders=[]
        if rgroup:
            for checkii in rgroup.buttons():
                if checkii.isChecked():
                    fid=self.button_le_dict[checkii]
                    # (folderid, foldername)
                    folders.append((fid, self.folder_dict[fid][0]))

        result_dict['folders_l']=folders
        LOGGER.debug('folders_l = %s' %folders)

        #-------------------Add new doc-------------------
        self.add_new_doc_sig.emit(result_dict)

        #---------------Del previous members---------------
        self.del_doc_sig.emit(members, False, False)

        # delete this group from tree
        del_header=self.group_dict[self.gid]['header']
        for ii in range(del_header.childCount()):
            cii=del_header.child(ii)
            del_header.removeChild(cii)
        self.tree.invisibleRootItem().removeChild(del_header)
        del self.group_dict[self.gid]

        #--------------Go back to table view--------------
        self.change_to_table_sig.emit()

        return

    @pyqtSlot()
    def previewMerge(self):
        pass


import pandas as pd
import matplotlib.pyplot as plt
import os
import numpy as np

def plot_csv_files(files_list, x_column_index, y_column_index, labels):
    # Initialize the figure and axis
    fig, ax = plt.subplots()

    # Define a list of colors for each line
    colors = ['r', 'm', 'b', 'c', 'g', 'y', 'k']

    # Loop through each CSV file and plot the data
    for i, (file, label) in enumerate(zip(files_list, labels)):
        file_path = os.path.join(r'C:\Users\ahj28\Desktop\Garcia DISC Data\ControlsAfterAugust', file)
        df = pd.read_csv(file_path, header=None)  # Skip header row with column names
        color = colors[i % len(colors)]  # Cycle through colors if more lines than colors
        ax.plot(df.iloc[:, x_column_index], df.iloc[:, y_column_index], label=label, color=color)

    # Set labels and legend
    ax.set_xlabel("Frames")
    ax.set_ylabel("Values")
    ax.legend()

    plt.vlines(x=np.linspace(0,98,11), ymin=0, ymax=len(np.linspace(1,6,4)), colors='purple', ls='--', lw=1)

    # Show the plot
    plt.show()

# Example usage with three files
#files_list = ['b3\\b3aft\graph\Right_Up.csv','b3\\b3aft\graph\Left_Up.csv','b3\\b3aft\graph\Left_Down.csv','b3\\b3aft\graph\Right_Down.csv']  # Add more filenames as needed
files_list = ['Ketamin\\3b\graph\Whole Face.csv','Ketamin\\4a\graph\Whole Face.csv']
x_column_index = 0  # Assuming the x values are in the first column (index 0)
y_column_index = 1  # Assuming the y values are in the second, third, and fourth columns (indices 1, 2, and 3)
#labels = ['right up', 'left up', 'left down', 'right down']
labels = ['3b','4a']
0
plot_csv_files(files_list, x_column_index, y_column_index, labels)


import requests
from lxml import etree
import os
import os.path
import pdfkit

root_dir = 'C:/Users/may/Documents/booknotes/'
path_wkthmltopdf = "C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"
pdfkit_config = pdfkit.configuration(wkhtmltopdf = path_wkthmltopdf)
pdfkit_options = {'javascript-delay': '10000',
                  'no-stop-slow-scripts': '',
                  }

def save_pdffile(url):
    parts = url.split('/')
    file_name = parts[-1].replace('html', 'pdf')
    dir = root_dir + parts[-2] + '/'
    if not os.path.exists(dir):
        os.makedirs(dir)
    file_name = dir + file_name

    if not os.path.exists(file_name):
        print("converting" + file_name)
        try:
            pdfkit.from_url(url, file_name, options=pdfkit_options, configuration=pdfkit_config)
        except Exception:
            pass

# save_pdffile('https://reading.geek-docs.com/literature/zimmerman-telegram.html')

if __name__ == '__main__':
    url = 'https://reading.geek-docs.com/sitemap.xml'
    # Pack the request parameters
    # Query String Parameters for a static page
    # For a dynamic content page using ajax query, filter XHR
    # could be in Form Data, then the data dictionary is used to set data parameter in the request
    param = {
        'type': '17',
        'interval_id': '100:90',
        'action': '',
        'start': '0',
        'limit': '20'
    }

    # UA fake pretend from a real browser
    # inspect the webpage -> Network -> resend a request by refreshing
    # pick the first request -> Headers -> Request Headers -> User-Agent
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'
    }

    # page = requests.get('https://reading.geek-docs.com/books/contagious-why-things-catch-on', params=param, headers=headers).text
    # tree = etree.HTML(page)
    # urls = tree.xpath('/html/body/section/div[1]/div/article/a/@href')
    # for u in urls:
    #     print(u)

    # General -> Request Method
    response = requests.get(url=url, params=param, headers=headers)

    with open('notes.txt', 'r') as f:
        fc = int(f.read())
    f.close()

    root = etree.fromstring(response.content)
    total = len(root)
    diff = total - fc

    if diff > 0:
        i = 1
        with open('notes.txt', 'w') as f:
            f.write(str(total))
        f.close()
        for u in root:
            if i <= diff:
                url = u[0].text
                print(str(i) + '/' + str(diff))
                if url.endswith('.html'):
                    save_pdffile(url)
                else:
                    page = requests.get(url, params=param, headers=headers).text
                    tree = etree.HTML(page)
                    urls = tree.xpath('/html/body/section/div[1]/div/article/a/@href')
                    for u in urls:
                        save_pdffile(u)
            i += 1


import mock
import errno
import json
import botocore.session

from awscli.customizations import sessionmanager
from awscli.testutils import unittest

class TestSessionManager(unittest.TestCase):

    def setUp(self):
        self.session = mock.Mock(botocore.session.Session)
        self.client = mock.Mock()
        self.region = 'us-west-2'
        self.profile = 'testProfile'
        self.endpoint_url = 'testUrl'
        self.client.meta.region_name = self.region
        self.client.meta.endpoint_url = self.endpoint_url
        self.session.create_client.return_value = self.client
        self.session.profile = self.profile
        self.caller = sessionmanager.StartSessionCaller(self.session)

    def test_start_session_when_non_custom_start_session_fails(self):
        self.client.start_session.side_effect = Exception('some exception')
        params = {}
        with self.assertRaisesRegexp(Exception, 'some exception'):
            self.caller.invoke('ssm', 'StartSession', params, mock.Mock())

    @mock.patch('awscli.customizations.sessionmanager.check_call')
    def test_start_session_success_scenario(self, mock_check_call):
        mock_check_call.return_value = 0

        start_session_params = {
            "Target": "i-123456789"
        }

        start_session_response = {
            "SessionId": "session-id",
            "TokenValue": "token-value",
            "StreamUrl": "stream-url"
        }

        self.client.start_session.return_value = start_session_response

        rc = self.caller.invoke('ssm', 'StartSession',
                                start_session_params, mock.Mock())
        self.assertEquals(rc, 0)
        self.client.start_session.assert_called_with(**start_session_params)
        mock_check_call_list = mock_check_call.call_args[0][0]
        mock_check_call_list[1] = json.loads(mock_check_call_list[1])
        self.assertEqual(
            mock_check_call_list,
            ['session-manager-plugin',
             start_session_response,
             self.region,
             'StartSession',
             self.profile,
             json.dumps(start_session_params),
             self.endpoint_url]
        )

    @mock.patch('awscli.customizations.sessionmanager.check_call')
    def test_start_session_when_check_call_fails(self, mock_check_call):
        mock_check_call.side_effect = OSError(errno.ENOENT, 'some error')

        start_session_params = {
            "Target": "i-123456789"
        }

        start_session_response = {
            "SessionId": "session-id",
            "TokenValue": "token-value",
            "StreamUrl": "stream-url"
        }

        terminate_session_params = {
            "SessionId": "session-id"
        }

        self.client.start_session.return_value = start_session_response

        with self.assertRaises(ValueError):
            self.caller.invoke('ssm', 'StartSession',
                               start_session_params, mock.Mock())

            self.client.start_session.assert_called_with(
                **start_session_params)
            self.client.terminate_session.assert_called_with(
                **terminate_session_params)

            mock_check_call_list = mock_check_call.call_args[0][0]
            mock_check_call_list[1] = json.loads(mock_check_call_list[1])
            self.assertEqual(
                mock_check_call_list,
                ['session-manager-plugin',
                 start_session_response,
                 self.region,
                 'StartSession',
                 self.profile,
                 json.dumps(start_session_params),
                 self.endpoint_url]
            )

    @mock.patch('awscli.customizations.sessionmanager.check_call')
    def test_start_session_when_no_profile_is_passed(self, mock_check_call):
        self.session.profile = None
        mock_check_call.return_value = 0

        start_session_params = {
            "Target": "i-123456789"
        }

        start_session_response = {
            "SessionId": "session-id",
            "TokenValue": "token-value",
            "StreamUrl": "stream-url"
        }

        self.client.start_session.return_value = start_session_response

        rc = self.caller.invoke('ssm', 'StartSession',
                                start_session_params, mock.Mock())
        self.assertEquals(rc, 0)
        self.client.start_session.assert_called_with(**start_session_params)
        mock_check_call_list = mock_check_call.call_args[0][0]
        self.assertEquals(mock_check_call_list[4], '')


# Databricks notebook source
# MAGIC %md
# MAGIC # Artificial Data Generator: Init Schemas

# COMMAND ----------

# MAGIC %md
# MAGIC ## Setup

# COMMAND ----------

# MAGIC %md
# MAGIC ### Imports

# COMMAND ----------

# MAGIC %run ./notebooks/common/widget_utils

# COMMAND ----------

# MAGIC %md
# MAGIC ### Widgets

# COMMAND ----------

if check_databricks():
  dbutils.widgets.removeAll()
  dbutils.widgets.text("db",            "iuod_artificial_data_generator",     "0.1 Database")
  dbutils.widgets.text("notebook_root", "iuod_artificial_data_generator/dev", "0.2 Notebook Root")
else:
  # Only make widgets in databricks
  pass

# COMMAND ----------

# MAGIC %md
# MAGIC ## Main

# COMMAND ----------

database_name = get_required_argument("db")
create_meta_notebook_path = "./schemas/create_meta_table"
open_data_metadata_uplift_path = "./schemas/uplifts/open_data_metadata_uplift"

notebook_params = [
  {
    "path": open_data_metadata_uplift_path,
    "timeout_seconds": 0,
    "arguments": {
      "database_name": database_name,
      "table_name": "artificial_hes_meta",
    }
  },
  {
    "path": create_meta_notebook_path,
    "timeout_seconds": 300,
    "arguments": {
      "database_name": database_name,
      "table_name": "artificial_hes_meta",
    }
  },
]

# COMMAND ----------

for params in notebook_params:
  dbutils.notebook.run(**params)

# COMMAND ----------


import time
from requests import get

class InstanceStateChecker:
    def __init__(self, service_ips):
        self.service_ips = service_ips

    def wait_for_services(self):
        responding = []
        not_responding = self.service_ips
        while True:
            for ip in not_responding:
                response = None
                try:
                    response = get("http://{}:8080/users/ids".format(ip), timeout=3)
                except Exception:
                    print("Timeout on {}".format(ip))
                if response is not None and response.status_code == 200:
                    responding.append(ip)
                    not_responding.remove(ip)
            self.print_status(responding, not_responding)

            if len(not_responding) == 0:
                break
            else:
                time.sleep(1)

    def print_status(self, responding, not_responding):
        print("###############################################################")
        print("Instance state checker report:")
        for i in responding:
            print("Responding: {}".format(i))
        for i in not_responding:
            print("Not responding: {}".format(i))
        print("###############################################################")


import pandas as pd
import matplotlib.pyplot as plt
from preprocessing import Preprocessor
from sentiment import SentimentClassifier
import datetime
import numpy as np

def pie_chart(comments):
    df = pd.DataFrame.from_dict(comments)
    sentiment_classifier = SentimentClassifier()
    df = sentiment_classifier.clear_data_frame(df)
    preprocessor = Preprocessor()
    df = preprocessor.create_sentences_column(df)
    return __create_pie_chart(df, sentiment_classifier)

def __create_pie_chart(sentences, sentiment_classifier):
    sentences['sentiment'], sentences['intensity'] = zip(*sentences['sentences'].apply(sentiment_classifier.score_comment_sentiment))
    labels = ["Positive", "Negative", "Neutral"]
    sum = sentences['sentiment'].count()
    neg_count = sentences[sentences['sentiment'] == 'Negative']['sentiment'].count()
    neu_count = sentences[sentences['sentiment'] == 'Neutral']['sentiment'].count()
    pos_count = sentences[sentences['sentiment'] == 'Positive']['sentiment'].count()
    sizes = [pos_count/sum, neg_count/sum, neu_count/sum]

    fig1, ax1 = plt.subplots()
    ax1.pie(sizes, labels=labels, autopct='%1.1f%%',
            shadow=True, startangle=90)
    ax1.axis('equal')

    plt.savefig("./static/img/"+ sentences['videoId'].values[0]+".png", format='png')

    plt.figure()
    plt.gcf().subplots_adjust(bottom=0.15)
    sentences[sentences["intensity"] > 0.75].groupby('sentiment').intensity.count().plot(kind="bar", xlabel="Sentiment", ylabel="Number of high intensity comments")
    plt.savefig("./static/img/"+ sentences['videoId'].values[0]+"IntensityDist.png")

    sentences["date"] = sentences["publishedAt"].apply(__parse_date)
    del sentences['publishedAt']
    sentences['date'] = pd.to_datetime(sentences['date'], format='%Y-%m-%d')
    fig, ax = plt.subplots(figsize=(7,3))
    plt.gcf().subplots_adjust(bottom=0.1)
    sentences[['date', 'sentiment']][sentences['sentiment'] == 'Positive'].groupby('date').count().plot(ax=ax, color='green')
    sentences[['date', 'sentiment']][sentences['sentiment'] == 'Negative'].groupby('date').count().plot(ax=ax, color='red')
    ax.legend(['Positive', 'Negative'])
    ax.set_xlabel("Date")
    ax.set_ylabel("Number of comments posted")

    plt.savefig("./static/img/"+ sentences['videoId'].values[0]+"Dates.png", format='png')

    plt.figure(figsize=(7,3))
    plt.gcf().subplots_adjust(bottom=0.1)
    sentences.loc[(sentences['sentiment'] == 'Positive') | (sentences['sentiment'] == 'Negative')].groupby('date').filter(lambda x: x['intensity'].count() > 2).groupby('date').intensity.min().plot(xlabel="Date of comments posted", ylabel = "Average intensity of comments", yticks=np.arange(0, 1, 0.1))
    plt.savefig("./static/img/"+ sentences['videoId'].values[0]+"DatesIntesity.png", format='png')

    return "/static/img/"+ sentences['videoId'].values[0]+".png", "/static/img/"+ sentences['videoId'].values[0]+"IntensityDist.png", "/static/img/"+ sentences['videoId'].values[0]+"Dates.png", "/static/img/"+ sentences['videoId'].values[0]+"DatesIntesity.png"

def __parse_date(text):
    dateText = text.split('T')[0]
    return datetime.datetime.strptime(dateText, "%Y-%m-%d").date()


import os
import argparse
from BasicTools import parse_file

def concat_logs(log0_path, log1_path, result_log_path):

    if os.path.exists(result_log_path):
        raise Exception(f'{result_log_path} already exists')

    log0 = parse_file.file2dict(log0_path)
    log1 = parse_file.file2dict(log1_path)

    concat_log = {}
    for key in log0.keys():
        if log0[key] not in log1.keys():
            raise Exception(f'{log0[key]} not in {log1_path}')
        concat_log[key] = log1[log0[key]]
    parse_file.dict2file(concat_log, result_log_path)

def parse_arg():
    parser = argparse.ArgumentParser(description='parse argments')
    parser.add_argument('--log', dest='log_path', nargs='+',
                        required=True, type=str, help='')
    parser.add_argument('--result-log', dest='result_log_path',
                        required=True, type=str)
    args = parser.parse_args()
    return args

def main():
    args = parse_arg()
    concat_logs(args.log_path[0], args.log_path[1], args.result_log_path)

if __name__ == '__main__':
    main()


from django.conf import settings

from . import tasks

class AnalyticsMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        response = self.get_response(request)

        if not request.META.get('HTTP_DNT', '0') == '1':
            if not request.session or not request.session.session_key:
                request.session.save()

            page = {
                'agent': request.META.get('HTTP_USER_AGENT', ''),
                'path': request.path,
                'referrer': request.META.get('HTTP_REFERRER', ''),
                'session_key': request.session._get_or_create_session_key(),
                'is_authenticated': request.user.is_authenticated
            }

            if hasattr(settings, 'ANALYTICS_IGNORE_PATHS'):
                if not any(request.path.startswith(path) for path in settings.ANALYTICS_IGNORE_PATHS):
                    tasks.pageview_add_task.delay(**page)
            else:
                tasks.pageview_add_task.delay(**page)
        return response


import os
import sys
import json

from lxml import etree
import zipfile

def read_in():
    line = sys.stdin.readline()
    return json.loads(line)

def main():
    try:
        docId = read_in()

        ooXMLns = {
            'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
        docxZip = zipfile.ZipFile(
            './src/collections/docs/documents/' + docId + '.docx')

        fileContent = docxZip.namelist()
        extraction = {'comments': [], 'inserts': [],
                      'deletes': [], 'lines': []}

        if ('word/comments.xml' in fileContent):
            commentsXML = docxZip.read('word/comments.xml')
            et = etree.XML(commentsXML)

            comments = et.xpath('//w:comment', namespaces=ooXMLns)
            for c in comments:
                authors = c.xpath('@w:author', namespaces=ooXMLns)
                date = c.xpath('@w:date', namespaces=ooXMLns)
                content = c.xpath('string(.)', namespaces=ooXMLns)
                extraction['comments'].append({
                    'authors': authors,
                    'date': date,
                    'content': content
                })

        if ('word/document.xml' in fileContent):
            insertsXML = docxZip.read('word/document.xml')
            te = etree.XML(insertsXML)

            inserts = te.xpath('//w:p', namespaces=ooXMLns)
            x = 0
            y = 0
            for i in inserts:
                l = i.xpath('string(.)', namespaces=ooXMLns)
                if (l != ''):
                    extraction['lines'].append(l)
                if (i.xpath('w:ins', namespaces=ooXMLns)):
                    line = i.xpath('string(.)', namespaces=ooXMLns)
                    extraction['inserts'].append({
                        'line': line
                    })
                    ins = i.xpath('w:r|w:ins', namespaces=ooXMLns)
                    inserted = []
                    for ting in ins:
                        author = ting.xpath('@w:author', namespaces=ooXMLns)
                        date = ting.xpath('@w:date', namespaces=ooXMLns)
                        content = ting.xpath('string(.)', namespaces=ooXMLns)
                        inserted.append({
                            'author': author,
                            'date': date,
                            'content': content
                        })
                    extraction['inserts'][x]['inserted'] = inserted
                    x += 1
                if (i.xpath('w:del', namespaces=ooXMLns)):
                    line = i.xpath('string(.)', namespaces=ooXMLns)
                    extraction['deletes'].append({
                        'line': line
                    })
                    d = i.xpath('w:r|w:del', namespaces=ooXMLns)
                    deleted = []
                    for ting in d:
                        author = ting.xpath('@w:author', namespaces=ooXMLns)
                        date = ting.xpath('@w:date', namespaces=ooXMLns)
                        content = ting.xpath('string(.)', namespaces=ooXMLns)
                        deleted.append({
                            'author': author,
                            'date': date,
                            'content': content
                        })
                    extraction['deletes'][y]['deleted'] = deleted
                    y += 1

        jsonObj = json.dumps(extraction)
        print(jsonObj)

    except:
        error = json.dumps({'error': True, 'data': sys.exc_info()[0]})
        print(error)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

from binascii import hexlify
from PIL import Image

data = []

with Image.open("font.png") as im:
    pixels = list(im.getdata())

    for c in range(0, 256):
        for y in range(0, 16):
            byte = 0
            for x in range(0, 8):
                p = pixels[((c // 16) * 16 + y) * (8 * 16) + (c % 16) * 8 + x]
                byte >>= 1
                byte |= 0x80 if p[0] > 128 else 0x00
            data.append(byte)

with open("build/chr_rom.hex", "wb") as f:
    f.write(hexlify(bytes(data), b" "))


"""
Unit test for module ngrammodel

When run as a script, this module invokes several procedures that
test the various functions in the module ngrammodel.

Author: Tianli Xia
Date:   Sep 12th, 2018
"""

import ngrammodel
import introcs
import string

def testA():
    """
    test for ngrams functions.
    """
    print("Test the ngrams functions.")
    input="aa bb cc dd bb aa cc bb"
    output=ngrammodel.ngrams(input, 2)
    print(output)

def testB():
    """
    test for preprocessed functions.
    """
    print("Test the preprocessed functions.")
    input="I went to the school, and didn't see any one."
    output="i went to the school and didn't see any one"
    print(ngrammodel.prep(input, 1))
    introcs.assert_equals(ngrammodel.prep(input, 1), output)

testA()
testB()

print("Please see the return value to determine if this runs correctly.")