Examples with NativeBroker org.exist.storage.NativeBroker used on opensource projects

Example 1 with NativeBroker

use of org.exist.storage.NativeBroker in project exist by eXist-db.

the class ConsistencyCheck method checkXMLTree.

/**
 * Check the persistent DOM of a document. The method traverses the entire node tree and checks it for consistency, including node relationships,
 * child and attribute counts etc.
 *
 * @param doc the document to check
 * @return null if the document is consistent, an error report otherwise.
 */
public ErrorReport checkXMLTree(final DocumentImpl doc) {
    final DOMFile domDb = ((NativeBroker) broker).getDOMFile();
    return new DOMTransaction<ErrorReport>(this, domDb, () -> broker.getBrokerPool().getLockManager().acquireBtreeWriteLock(domDb.getLockName()), doc) {

        public ErrorReport start() {
            EmbeddedXMLStreamReader reader = null;
            try {
                final Node root = doc.getFirstChild();
                reader = (EmbeddedXMLStreamReader) broker.getXMLStreamReader((NodeHandle) root, true);
                boolean attribsAllowed = false;
                int expectedAttribs = 0;
                int attributeCount = 0;
                while (reader.hasNext()) {
                    final int status = reader.next();
                    final NodeId nodeId = (NodeId) reader.getProperty(EmbeddedXMLStreamReader.PROPERTY_NODE_ID);
                    if ((status != XMLStreamReader.END_ELEMENT) && !elementStack.isEmpty()) {
                        final ElementNode parent = elementStack.peek();
                        parent.childCount++;
                        // test parent-child relation
                        if (!nodeId.isChildOf(parent.elem.getNodeId())) {
                            return new ErrorReport.ResourceError(ErrorReport.NODE_HIERARCHY, "Node " + nodeId + " is not a child of " + parent.elem.getNodeId());
                        }
                        // test sibling relation
                        if ((parent.prevSibling != null) && !(nodeId.isSiblingOf(parent.prevSibling) && (nodeId.compareTo(parent.prevSibling) > 0))) {
                            return new ErrorReport.ResourceError(ErrorReport.INCORRECT_NODE_ID, "Node " + nodeId + " is not a sibling of " + parent.prevSibling);
                        }
                        parent.prevSibling = nodeId;
                    }
                    switch(status) {
                        case XMLStreamReader.ATTRIBUTE:
                            {
                                attributeCount++;
                                break;
                            }
                        case XMLStreamReader.END_ELEMENT:
                            {
                                if (elementStack.isEmpty()) {
                                    return new org.exist.backup.ErrorReport.ResourceError(ErrorReport.NODE_HIERARCHY, "Error in node hierarchy: received END_ELEMENT event " + "but stack was empty!");
                                }
                                final ElementNode lastElem = elementStack.pop();
                                if (lastElem.childCount != lastElem.elem.getChildCount()) {
                                    return new ErrorReport.ResourceError(org.exist.backup.ErrorReport.NODE_HIERARCHY, "Element reports incorrect child count: expected " + lastElem.elem.getChildCount() + " but found " + lastElem.childCount);
                                }
                                break;
                            }
                        case XMLStreamReader.START_ELEMENT:
                            {
                                if (nodeId.getTreeLevel() <= defaultIndexDepth) {
                                    // check dom.dbx btree, which maps the node
                                    // id to the node's storage address
                                    // look up the node id and check if the
                                    // returned storage address is correct
                                    final NativeBroker.NodeRef nodeRef = new NativeBroker.NodeRef(doc.getDocId(), nodeId);
                                    try {
                                        final long p = domDb.findValue(nodeRef);
                                        if (p != reader.getCurrentPosition()) {
                                            final Value v = domDb.get(p);
                                            if (v == null) {
                                                return new ErrorReport.IndexError(ErrorReport.DOM_INDEX, "Failed to access node " + nodeId + " through dom.dbx index. Wrong storage address. Expected: " + p + "; got: " + reader.getCurrentPosition() + " - ", doc.getDocId());
                                            }
                                        }
                                    } catch (final Exception e) {
                                        e.printStackTrace();
                                        return new ErrorReport.IndexError(ErrorReport.DOM_INDEX, "Failed to access node " + nodeId + " through dom.dbx index.", e, doc.getDocId());
                                    }
                                }
                                final IStoredNode node = reader.getNode();
                                if (node.getNodeType() != Node.ELEMENT_NODE) {
                                    return new org.exist.backup.ErrorReport.ResourceError(ErrorReport.INCORRECT_NODE_TYPE, "Expected an element node, received node of type " + node.getNodeType());
                                }
                                elementStack.push(new ElementNode((ElementImpl) node));
                                attribsAllowed = true;
                                attributeCount = 0;
                                expectedAttribs = reader.getAttributeCount();
                                break;
                            }
                        default:
                            {
                                if (attribsAllowed) {
                                    if (attributeCount != expectedAttribs) {
                                        return new org.exist.backup.ErrorReport.ResourceError(ErrorReport.INCORRECT_NODE_TYPE, "Wrong number of attributes. Expected: " + expectedAttribs + "; found: " + attributeCount);
                                    }
                                }
                                attribsAllowed = false;
                                break;
                            }
                    }
                }
                if (!elementStack.isEmpty()) {
                    return new org.exist.backup.ErrorReport.ResourceError(ErrorReport.NODE_HIERARCHY, "Error in node hierarchy: reached end of tree but " + "stack was not empty!");
                }
                return null;
            } catch (final IOException | XMLStreamException e) {
                e.printStackTrace();
                return new org.exist.backup.ErrorReport.ResourceError(ErrorReport.RESOURCE_ACCESS_FAILED, e.getMessage(), e);
            } finally {
                elementStack.clear();
                if (reader != null) {
                    try {
                        reader.close();
                    } catch (final XMLStreamException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }.run();
}

Example 2 with NativeBroker

use of org.exist.storage.NativeBroker in project exist by eXist-db.

the class DOMFile method debugPageContents.

String debugPageContents(final DOMPage page) {
    final StringBuilder buf = new StringBuilder();
    buf.append("Page ").append(page.getPageNum()).append(": ");
    short count = 0;
    final int dataLength = page.getPageHeader().getDataLength();
    for (int pos = 0; pos < dataLength; count++) {
        buf.append(pos).append("/");
        final short tupleID = ByteConversion.byteToShort(page.data, pos);
        pos += LENGTH_TID;
        buf.append(ItemId.getId(tupleID));
        if (ItemId.isLink(tupleID)) {
            buf.append("L");
        } else if (ItemId.isRelocated(tupleID)) {
            buf.append("R");
        }
        if (ItemId.isLink(tupleID)) {
            final long forwardLink = ByteConversion.byteToLong(page.data, pos);
            buf.append(':').append(forwardLink).append(" ");
            pos += LENGTH_FORWARD_LOCATION;
        } else {
            final short valueLength = ByteConversion.byteToShort(page.data, pos);
            pos += LENGTH_DATA_LENGTH;
            if (valueLength < 0) {
                LOG.warn("Illegal length: {}", valueLength);
                return buf.append("[Illegal length : ").append(valueLength).append("] ").toString();
            // Probably unable to continue...
            } else if (ItemId.isRelocated(tupleID)) {
                // TODO : output to buffer ?
                pos += LENGTH_ORIGINAL_LOCATION;
            } else {
                buf.append("[");
                switch(Signatures.getType(page.data[pos])) {
                    case Node.ELEMENT_NODE:
                        {
                            buf.append("element ");
                            int readOffset = pos;
                            readOffset += 1;
                            final int children = ByteConversion.byteToInt(page.data, readOffset);
                            readOffset += ElementImpl.LENGTH_ELEMENT_CHILD_COUNT;
                            final int dlnLen = ByteConversion.byteToShort(page.data, readOffset);
                            readOffset += NodeId.LENGTH_NODE_ID_UNITS;
                            // That might happen during recovery runs : TODO, investigate
                            if (owner == null) {
                                buf.append("(Can't read data, owner is null)");
                            } else {
                                try {
                                    final NodeId nodeId = ((NativeBroker) owner).getBrokerPool().getNodeFactory().createFromData(dlnLen, page.data, readOffset);
                                    readOffset += nodeId.size();
                                    buf.append("(").append(nodeId.toString()).append(")");
                                    final short attributes = ByteConversion.byteToShort(page.data, readOffset);
                                    buf.append(" children: ").append(children);
                                    buf.append(" attributes: ").append(attributes);
                                } catch (final Exception e) {
                                    // TODO : more friendly message. Provide the array of bytes ?
                                    buf.append("(Unable to read the node ID at: ").append(readOffset);
                                    buf.append(" children : ").append(children);
                                    // Probably a wrong offset so... don't read it
                                    buf.append(" attributes : unknown");
                                }
                            }
                            break;
                        }
                    case Node.TEXT_NODE:
                    case Node.CDATA_SECTION_NODE:
                        {
                            if (Signatures.getType(page.data[pos]) == Node.TEXT_NODE) {
                                buf.append("text ");
                            } else {
                                buf.append("CDATA ");
                            }
                            int readOffset = pos;
                            readOffset += 1;
                            final int dlnLen = ByteConversion.byteToShort(page.data, readOffset);
                            readOffset += NodeId.LENGTH_NODE_ID_UNITS;
                            // That might happen during recovery runs : TODO, investigate
                            if (owner == null) {
                                buf.append("(Can't read data, owner is null)");
                            } else {
                                try {
                                    final NodeId nodeId = ((NativeBroker) owner).getBrokerPool().getNodeFactory().createFromData(dlnLen, page.data, readOffset);
                                    readOffset += nodeId.size();
                                    buf.append("(").append(nodeId.toString()).append(")");
                                    String value = new String(page.data, readOffset, valueLength - (readOffset - pos), UTF_8);
                                    if (value.length() > 15) {
                                        value = value.substring(0, 8) + "..." + value.substring(value.length() - 8);
                                    }
                                    buf.append(":'").append(value).append("'");
                                } catch (final Exception e) {
                                    // TODO : more friendly message. Provide the array of bytes ?
                                    buf.append("(unable to read the node ID at : ").append(readOffset);
                                }
                            }
                            break;
                        }
                    case Node.ATTRIBUTE_NODE:
                        {
                            buf.append("[");
                            buf.append("attribute ");
                            int readOffset = pos;
                            final byte idSizeType = (byte) (page.data[readOffset] & 0x3);
                            final boolean hasNamespace = (page.data[readOffset] & 0x10) == 0x10;
                            readOffset += 1;
                            final int dlnLen = ByteConversion.byteToShort(page.data, readOffset);
                            readOffset += NodeId.LENGTH_NODE_ID_UNITS;
                            // That might happen during recovery runs : TODO, investigate
                            if (owner == null) {
                                buf.append("(can't read data, owner is null)");
                            } else {
                                try {
                                    final NodeId nodeId = ((NativeBroker) owner).getBrokerPool().getNodeFactory().createFromData(dlnLen, page.data, readOffset);
                                    readOffset += nodeId.size();
                                    buf.append("(").append(nodeId.toString()).append(")");
                                    readOffset += Signatures.getLength(idSizeType);
                                    if (hasNamespace) {
                                        // Untested
                                        final short NSId = ByteConversion.byteToShort(page.data, readOffset);
                                        readOffset += AttrImpl.LENGTH_NS_ID;
                                        final short prefixLen = ByteConversion.byteToShort(page.data, readOffset);
                                        readOffset += AttrImpl.LENGTH_PREFIX_LENGTH + prefixLen;
                                        final String prefix = new String(page.data, readOffset, valueLength - (readOffset - prefixLen), UTF_8);
                                        final String NsURI = ((NativeBroker) owner).getBrokerPool().getSymbols().getNamespace(NSId);
                                        buf.append(prefix).append("{").append(NsURI).append("}");
                                    }
                                    String value = new String(page.data, readOffset, valueLength - (readOffset - pos), UTF_8);
                                    if (value.length() > 15) {
                                        value = value.substring(0, 8) + "..." + value.substring(value.length() - 8);
                                    }
                                    buf.append(":'").append(value).append("'");
                                } catch (final Exception e) {
                                    // TODO : more friendly message. Provide the array of bytes ?
                                    buf.append("(unable to read the node ID at : ").append(readOffset);
                                }
                            }
                            buf.append("] ");
                            break;
                        }
                    default:
                        buf.append("Unknown node type !");
                }
                buf.append("] ");
            }
            pos += valueLength;
        }
    }
    buf.append("; records in page: ").append(count).append(" (header says: ").append(page.getPageHeader().getRecordCount()).append(")");
    buf.append("; currentTupleID: ").append(page.getPageHeader().getCurrentTupleID());
    buf.append("; data length: ").append(page.getPageHeader().getDataLength());
    for (int i = page.data.length; i > 0; i--) {
        if (page.data[i - 1] != 0) {
            buf.append(" (last non-zero byte: ").append(i).append(")");
            break;
        }
    }
    return buf.toString();
}

Example 3 with NativeBroker

use of org.exist.storage.NativeBroker in project exist by eXist-db.

the class Repair method repair.

public void repair(String id) {
    try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) {
        BTree btree = null;
        if ("collections".equals(id)) {
            btree = ((NativeBroker) broker).getStorage(NativeBroker.COLLECTIONS_DBX_ID);
        } else if ("dom".equals(id)) {
            btree = ((NativeBroker) broker).getStorage(NativeBroker.DOM_DBX_ID);
        } else if ("range".equals(id)) {
            btree = ((NativeBroker) broker).getStorage(NativeBroker.VALUES_DBX_ID);
        } else if ("structure".equals(id)) {
            NativeStructuralIndexWorker index = (NativeStructuralIndexWorker) broker.getIndexController().getWorkerByIndexName(StructuralIndex.STRUCTURAL_INDEX_ID);
            btree = index.getStorage();
        } else {
            // use index id defined in conf.xml
            Index index = pool.getIndexManager().getIndexByName(id);
            if (index != null) {
                btree = index.getStorage();
            }
        }
        if (btree == null) {
            System.console().printf("Unkown index: %s\n", id);
            return;
        }
        final LockManager lockManager = broker.getBrokerPool().getLockManager();
        try (final ManagedLock<ReentrantLock> btreeLock = lockManager.acquireBtreeWriteLock(btree.getLockName())) {
            System.console().printf("Rebuilding %15s ...", FileUtils.fileName(btree.getFile()));
            btree.rebuild();
            System.out.println("Done");
        }
    } catch (Exception e) {
        System.console().printf("An exception occurred during repair: %s\n", e.getMessage());
        e.printStackTrace();
    }
}