Search in sources :

Example 46 with DocumentSet

use of org.exist.dom.persistent.DocumentSet in project exist by eXist-db.

the class LuceneIndexTest method xupdateUpdate.

@Test
public void xupdateUpdate() throws EXistException, CollectionConfigurationException, PermissionDeniedException, SAXException, LockException, IOException, XPathException, ParserConfigurationException, QName.IllegalQNameException {
    final DocumentSet docs = configureAndStore(COLLECTION_CONFIG2, XML2, "xupdate.xml");
    final BrokerPool pool = existEmbeddedServer.getBrokerPool();
    final TransactionManager transact = pool.getTransactionManager();
    try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()));
        final Txn transaction = transact.beginTransaction()) {
        final Occurrences[] occur = checkIndex(docs, broker, new QName[] { new QName("description") }, "chair", 1);
        assertEquals("chair", occur[0].getTerm());
        checkIndex(docs, broker, new QName[] { new QName("item") }, null, 5);
        final XQuery xquery = pool.getXQueryService();
        assertNotNull(xquery);
        Sequence seq = xquery.execute(broker, "//item[ft:query(description, 'chair')]", null);
        assertNotNull(seq);
        assertEquals(1, seq.getItemCount());
        // Update element content
        final XUpdateProcessor proc = new XUpdateProcessor(broker, docs);
        proc.setBroker(broker);
        proc.setDocumentSet(docs);
        String xupdate = XUPDATE_START + "   <xu:update select=\"//item[@id = '1']/description\">wardrobe</xu:update>" + XUPDATE_END;
        Modification[] modifications = proc.parse(new InputSource(new StringReader(xupdate)));
        assertNotNull(modifications);
        modifications[0].process(transaction);
        proc.reset();
        checkIndex(docs, broker, new QName[] { new QName("description") }, null, 3);
        checkIndex(docs, broker, new QName[] { new QName("item") }, null, 5);
        checkIndex(docs, broker, new QName[] { new QName("description") }, "chair", 0);
        checkIndex(docs, broker, new QName[] { new QName("item") }, "chair", 0);
        Occurrences[] o = checkIndex(docs, broker, new QName[] { new QName("description") }, "wardrobe", 1);
        assertEquals("wardrobe", o[0].getTerm());
        // Update text node
        proc.setBroker(broker);
        proc.setDocumentSet(docs);
        xupdate = XUPDATE_START + "   <xu:update select=\"//item[@id = '1']/description/text()\">Wheelchair</xu:update>" + XUPDATE_END;
        modifications = proc.parse(new InputSource(new StringReader(xupdate)));
        assertNotNull(modifications);
        modifications[0].process(transaction);
        proc.reset();
        checkIndex(docs, broker, new QName[] { new QName("description") }, null, 3);
        checkIndex(docs, broker, new QName[] { new QName("item") }, null, 5);
        checkIndex(docs, broker, new QName[] { new QName("description") }, "wardrobe", 0);
        checkIndex(docs, broker, new QName[] { new QName("item") }, "wardrobe", 0);
        o = checkIndex(docs, broker, new QName[] { new QName("description") }, "wheelchair", 1);
        assertEquals("wheelchair", o[0].getTerm());
        // Update attribute value
        proc.setBroker(broker);
        proc.setDocumentSet(docs);
        xupdate = XUPDATE_START + "   <xu:update select=\"//item[@id = '1']/@attr\">abc</xu:update>" + XUPDATE_END;
        modifications = proc.parse(new InputSource(new StringReader(xupdate)));
        assertNotNull(modifications);
        modifications[0].process(transaction);
        proc.reset();
        final QName[] qnattr = { new QName("attr", XMLConstants.NULL_NS_URI, XMLConstants.DEFAULT_NS_PREFIX, ElementValue.ATTRIBUTE) };
        o = checkIndex(docs, broker, qnattr, null, 1);
        assertEquals("abc", o[0].getTerm());
        checkIndex(docs, broker, qnattr, "attribute", 0);
        transact.commit(transaction);
    }
}
Also used : XUpdateProcessor(org.exist.xupdate.XUpdateProcessor) Modification(org.exist.xupdate.Modification) InputSource(org.xml.sax.InputSource) QName(org.exist.dom.QName) XQuery(org.exist.xquery.XQuery) CompiledXQuery(org.exist.xquery.CompiledXQuery) Txn(org.exist.storage.txn.Txn) Sequence(org.exist.xquery.value.Sequence) DBBroker(org.exist.storage.DBBroker) TransactionManager(org.exist.storage.txn.TransactionManager) StringReader(java.io.StringReader) DefaultDocumentSet(org.exist.dom.persistent.DefaultDocumentSet) DocumentSet(org.exist.dom.persistent.DocumentSet) MutableDocumentSet(org.exist.dom.persistent.MutableDocumentSet) BrokerPool(org.exist.storage.BrokerPool)

Example 47 with DocumentSet

use of org.exist.dom.persistent.DocumentSet in project exist by eXist-db.

the class Query method preSelect.

public NodeSet preSelect(Sequence contextSequence, boolean useContext) throws XPathException {
    // guard against an empty contextSequence
    if (contextSequence == null || !contextSequence.isPersistentSet()) {
        // in-memory docs won't have an index
        return NodeSet.EMPTY_SET;
    }
    long start = System.currentTimeMillis();
    // the expression can be called multiple times, so we need to clear the previous preselectResult
    preselectResult = null;
    LuceneIndexWorker index = (LuceneIndexWorker) context.getBroker().getIndexController().getWorkerByIndexId(LuceneIndex.ID);
    DocumentSet docs = contextSequence.getDocumentSet();
    Item key = getKey(contextSequence, null);
    List<QName> qnames = new ArrayList<>(1);
    qnames.add(contextQName);
    QueryOptions options = parseOptions(this, contextSequence, null, 3);
    try {
        if (key != null && Type.subTypeOf(key.getType(), Type.ELEMENT)) {
            final Element queryXML = key == null ? null : (Element) ((NodeValue) key).getNode();
            preselectResult = index.query(getExpressionId(), docs, useContext ? contextSequence.toNodeSet() : null, qnames, queryXML, NodeSet.DESCENDANT, options);
        } else {
            final String query = key == null ? null : key.getStringValue();
            preselectResult = index.query(getExpressionId(), docs, useContext ? contextSequence.toNodeSet() : null, qnames, query, NodeSet.DESCENDANT, options);
        }
    } catch (IOException | org.apache.lucene.queryparser.classic.ParseException e) {
        throw new XPathException(this, "Error while querying full text index: " + e.getMessage(), e);
    }
    LOG.trace("Lucene query took {}", System.currentTimeMillis() - start);
    if (context.getProfiler().traceFunctions()) {
        context.getProfiler().traceIndexUsage(context, "lucene", this, PerformanceStats.OPTIMIZED_INDEX, System.currentTimeMillis() - start);
    }
    return preselectResult;
}
Also used : QName(org.exist.dom.QName) Element(org.w3c.dom.Element) ArrayList(java.util.ArrayList) IOException(java.io.IOException) LuceneIndexWorker(org.exist.indexing.lucene.LuceneIndexWorker) DocumentSet(org.exist.dom.persistent.DocumentSet)

Example 48 with DocumentSet

use of org.exist.dom.persistent.DocumentSet in project exist by eXist-db.

the class GeneralComparison method quickNodeSetCompare.

/**
 * Optimized implementation: first checks if a range index is defined on the nodes in the left argument.
 * Otherwise, fall back to {@link #nodeSetCompare(NodeSet, Sequence)}.
 *
 * @param   contextSequence  DOCUMENT ME!
 *
 * @return  DOCUMENT ME!
 *
 * @throws  XPathException  DOCUMENT ME!
 */
protected Sequence quickNodeSetCompare(Sequence contextSequence) throws XPathException {
    if (context.getProfiler().isEnabled()) {
        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION CHOICE", "quickNodeSetCompare");
    }
    final long start = System.currentTimeMillis();
    // get the NodeSet on the left
    final Sequence leftSeq = getLeft().eval(contextSequence);
    if (!leftSeq.isPersistentSet()) {
        return (genericCompare(leftSeq, contextSequence, null));
    }
    final NodeSet nodes = leftSeq.isEmpty() ? NodeSet.EMPTY_SET : (NodeSet) leftSeq;
    // nothing on the left, so nothing to do
    if (!(nodes instanceof VirtualNodeSet) && nodes.isEmpty()) {
        // Well, we might discuss this one ;-)
        hasUsedIndex = true;
        return (Sequence.EMPTY_SEQUENCE);
    }
    // get the Sequence on the right
    final Sequence rightSeq = getRight().eval(contextSequence);
    // nothing on the right, so nothing to do
    if (rightSeq.isEmpty()) {
        // Well, we might discuss this one ;-)
        hasUsedIndex = true;
        return (Sequence.EMPTY_SEQUENCE);
    }
    // get the type of a possible index
    final int indexType = nodes.getIndexType();
    // remember that Type.ITEM means... no index ;-)
    if (indexType != Type.ITEM) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("found an index of type: {}", Type.getTypeName(indexType));
        }
        boolean indexScan = false;
        boolean indexMixed = false;
        QName myContextQName = contextQName;
        if (contextSequence != null) {
            final IndexFlags iflags = checkForQNameIndex(idxflags, context, contextSequence, myContextQName);
            boolean indexFound = false;
            if (!iflags.indexOnQName) {
                // if myContextQName != null and no index is defined on
                // myContextQName, we don't need to scan other QName indexes
                // and can just use the generic range index
                indexFound = myContextQName != null;
                if (iflags.partialIndexOnQName) {
                    indexMixed = true;
                } else {
                    // set myContextQName to null so the index lookup below is not
                    // restricted to that QName
                    myContextQName = null;
                }
            }
            if (!indexFound && (myContextQName == null)) {
                // we need to check them all
                if (iflags.hasIndexOnQNames) {
                    indexScan = true;
                }
            // else use range index defined on path by default
            }
        } else {
            return (nodeSetCompare(nodes, contextSequence));
        }
        // Get the documents from the node set
        final DocumentSet docs = nodes.getDocumentSet();
        // Holds the result
        NodeSet result = null;
        // Iterate through the right hand sequence
        for (final SequenceIterator itRightSeq = Atomize.atomize(rightSeq).iterate(); itRightSeq.hasNext(); ) {
            // Get the index key
            Item key = itRightSeq.nextItem();
            // if key has truncation, convert it to string
            if (truncation != StringTruncationOperator.NONE) {
                if (!Type.subTypeOf(key.getType(), Type.STRING)) {
                    LOG.info("Truncated key. Converted from {} to xs:string", Type.getTypeName(key.getType()));
                    // truncation is only possible on strings
                    key = key.convertTo(Type.STRING);
                }
            } else // TODO : use Type.isSubType() ??? -pb
            if (key.getType() != indexType) {
                // try to convert the key to the index type
                try {
                    key = key.convertTo(indexType);
                } catch (final XPathException xpe) {
                    // Could not convert the key to a suitable type for the index, fallback to nodeSetCompare()
                    if (context.getProfiler().isEnabled()) {
                        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (" + xpe.getMessage() + ")");
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Cannot convert key: {} to required index type: {}", Type.getTypeName(key.getType()), Type.getTypeName(indexType));
                    }
                    return (nodeSetCompare(nodes, contextSequence));
                }
            }
            // If key implements org.exist.storage.Indexable, we can use the index
            if (key instanceof Indexable) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Checking if range index can be used for key: {}", key.getStringValue());
                }
                final Collator collator = ((collationArg != null) ? getCollator(contextSequence) : null);
                if (Type.subTypeOf(key.getType(), indexType)) {
                    if (truncation == StringTruncationOperator.NONE) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Using range index for key: {}", key.getStringValue());
                        }
                        // key without truncation, find key
                        context.getProfiler().message(this, Profiler.OPTIMIZATIONS, "OPTIMIZATION", "Using value index '" + context.getBroker().getValueIndex().toString() + "' to find key '" + Type.getTypeName(key.getType()) + "(" + key.getStringValue() + ")'");
                        NodeSet ns;
                        if (indexScan) {
                            ns = context.getBroker().getValueIndex().findAll(context.getWatchDog(), relation, docs, nodes, NodeSet.ANCESTOR, (Indexable) key);
                        } else {
                            ns = context.getBroker().getValueIndex().find(context.getWatchDog(), relation, docs, nodes, NodeSet.ANCESTOR, myContextQName, (Indexable) key, indexMixed);
                        }
                        hasUsedIndex = true;
                        if (result == null) {
                            result = ns;
                        } else {
                            result = result.union(ns);
                        }
                    } else {
                        // key with truncation, match key
                        if (LOG.isTraceEnabled()) {
                            context.getProfiler().message(this, Profiler.OPTIMIZATIONS, "OPTIMIZATION", "Using value index '" + context.getBroker().getValueIndex().toString() + "' to match key '" + Type.getTypeName(key.getType()) + "(" + key.getStringValue() + ")'");
                        }
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Using range index for key: {}", key.getStringValue());
                        }
                        try {
                            NodeSet ns;
                            final String matchString = key.getStringValue();
                            final int matchType = getMatchType(truncation);
                            if (indexScan) {
                                ns = context.getBroker().getValueIndex().matchAll(context.getWatchDog(), docs, nodes, NodeSet.ANCESTOR, matchString, matchType, 0, true, collator, truncation);
                            } else {
                                ns = context.getBroker().getValueIndex().match(context.getWatchDog(), docs, nodes, NodeSet.ANCESTOR, matchString, myContextQName, matchType, collator, truncation);
                            }
                            hasUsedIndex = true;
                            if (result == null) {
                                result = ns;
                            } else {
                                result = result.union(ns);
                            }
                        } catch (final EXistException e) {
                            throw (new XPathException(this, e));
                        }
                    }
                } else {
                    // our key does is not of the correct type
                    if (context.getProfiler().isEnabled()) {
                        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (key is of type: " + Type.getTypeName(key.getType()) + ") whereas index is of type '" + Type.getTypeName(indexType) + "'");
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Cannot use range index: key is of type: {}) whereas index is of type '{}", Type.getTypeName(key.getType()), Type.getTypeName(indexType));
                    }
                    return (nodeSetCompare(nodes, contextSequence));
                }
            } else {
                // our key does not implement org.exist.storage.Indexable
                if (context.getProfiler().isEnabled()) {
                    context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (key is not an indexable type: " + key.getClass().getName());
                }
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Cannot use key which is of type '{}", key.getClass().getName());
                }
                return (nodeSetCompare(nodes, contextSequence));
            }
        }
        if (context.getProfiler().traceFunctions()) {
            context.getProfiler().traceIndexUsage(context, PerformanceStats.RANGE_IDX_TYPE, this, PerformanceStats.BASIC_INDEX, System.currentTimeMillis() - start);
        }
        return (result);
    } else {
        if (LOG.isTraceEnabled()) {
            LOG.trace("No suitable index found for key: {}", rightSeq.getStringValue());
        }
        // no range index defined on the nodes in this sequence, so fallback to nodeSetCompare
        if (context.getProfiler().isEnabled()) {
            context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "falling back to nodeSetCompare (no index available)");
        }
        return (nodeSetCompare(nodes, contextSequence));
    }
}
Also used : NodeSet(org.exist.dom.persistent.NodeSet) VirtualNodeSet(org.exist.dom.persistent.VirtualNodeSet) NewArrayNodeSet(org.exist.dom.persistent.NewArrayNodeSet) QName(org.exist.dom.QName) Sequence(org.exist.xquery.value.Sequence) EXistException(org.exist.EXistException) VirtualNodeSet(org.exist.dom.persistent.VirtualNodeSet) Collator(com.ibm.icu.text.Collator) Item(org.exist.xquery.value.Item) ContextItem(org.exist.dom.persistent.ContextItem) SequenceIterator(org.exist.xquery.value.SequenceIterator) Indexable(org.exist.storage.Indexable) DocumentSet(org.exist.dom.persistent.DocumentSet)

Example 49 with DocumentSet

use of org.exist.dom.persistent.DocumentSet in project exist by eXist-db.

the class Modification method selectAndLock.

/**
 * Acquire a lock on all documents processed by this modification.
 * We have to avoid that node positions change during the
 * operation.
 *
 * @param nodes sequence containing nodes from documents to lock
 * @param transaction current transaction
 * @return array of nodes for which lock was acquired
 *
 * @throws LockException in case locking failed
 * @throws TriggerException in case of error thrown by triggers
 * @throws XPathException in case of dynamic error
 */
protected StoredNode[] selectAndLock(Txn transaction, Sequence nodes) throws LockException, XPathException, TriggerException {
    final java.util.concurrent.locks.Lock globalLock = context.getBroker().getBrokerPool().getGlobalUpdateLock();
    globalLock.lock();
    try {
        final DocumentSet lockedDocuments = nodes.getDocumentSet();
        // acquire a lock on all documents
        // we have to avoid that node positions change
        // during the modification
        lockedDocumentsLocks = lockedDocuments.lock(context.getBroker(), true);
        final StoredNode[] ql = new StoredNode[nodes.getItemCount()];
        for (int i = 0; i < ql.length; i++) {
            final Item item = nodes.itemAt(i);
            if (!Type.subTypeOf(item.getType(), Type.NODE)) {
                throw new XPathException(this, "XQuery update expressions can only be applied to nodes. Got: " + item.getStringValue());
            }
            final NodeValue nv = (NodeValue) item;
            if (nv.getImplementationType() == NodeValue.IN_MEMORY_NODE) {
                throw new XPathException(this, "XQuery update expressions can not be applied to in-memory nodes.");
            }
            final Node n = nv.getNode();
            if (n.getNodeType() == Node.DOCUMENT_NODE) {
                throw new XPathException(this, "Updating the document object is not allowed.");
            }
            ql[i] = (StoredNode) n;
            final DocumentImpl doc = ql[i].getOwnerDocument();
            // prepare Trigger
            prepareTrigger(transaction, doc);
        }
        return ql;
    } finally {
        globalLock.unlock();
    }
}
Also used : Item(org.exist.xquery.value.Item) NodeValue(org.exist.xquery.value.NodeValue) StoredNode(org.exist.dom.persistent.StoredNode) Node(org.w3c.dom.Node) DefaultDocumentSet(org.exist.dom.persistent.DefaultDocumentSet) DocumentSet(org.exist.dom.persistent.DocumentSet) MutableDocumentSet(org.exist.dom.persistent.MutableDocumentSet) DocumentImpl(org.exist.dom.persistent.DocumentImpl) StoredNode(org.exist.dom.persistent.StoredNode)

Example 50 with DocumentSet

use of org.exist.dom.persistent.DocumentSet in project exist by eXist-db.

the class HTTPUtils method addLastModifiedHeader.

/**
 * Feature "Guess last modification time for an XQuery result";
 *  the HTTP header Last-Modified is filled with most recent time stamp among all
 *  XQuery documents appearing in the actual response.
 *  Note however, that the actual response can be influenced, through tests in the query,
 *  by documents more recent.
 *
 * @param result the XQuery result to inspect
 * @param context current context
 */
public static void addLastModifiedHeader(Sequence result, XQueryContext context) {
    try {
        final DocumentSet documentSet = result.getDocumentSet();
        long mostRecentDocumentTime = 0;
        for (final Iterator<DocumentImpl> i = documentSet.getDocumentIterator(); i.hasNext(); ) {
            final DocumentImpl doc = i.next();
            if (doc != null) {
                mostRecentDocumentTime = Math.max(doc.getLastModified(), mostRecentDocumentTime);
            // LOG.debug("getFileName: " + doc.getFileName() + ", "
            // + doc.getLastModified());
            }
        }
        LOG.debug("mostRecentDocumentTime: {}", mostRecentDocumentTime);
        if (mostRecentDocumentTime > 0) {
            final Optional<ResponseWrapper> maybeResponse = Optional.ofNullable(context.getHttpContext()).map(XQueryContext.HttpContext::getResponse);
            if (maybeResponse.isPresent()) {
                // have to take in account that if the header has allready been explicitely set
                // by the XQuery script, we should not modify it .
                final ResponseWrapper responseWrapper = maybeResponse.get();
                if (responseWrapper.getDateHeader("Last-Modified") == 0) {
                    responseWrapper.setDateHeader("Last-Modified", mostRecentDocumentTime);
                }
            }
        }
    } catch (final Exception e) {
        LOG.debug(e.getMessage(), e);
    }
}
Also used : ResponseWrapper(org.exist.http.servlets.ResponseWrapper) DocumentSet(org.exist.dom.persistent.DocumentSet) DocumentImpl(org.exist.dom.persistent.DocumentImpl)

Aggregations

DocumentSet (org.exist.dom.persistent.DocumentSet)50 QName (org.exist.dom.QName)20 DefaultDocumentSet (org.exist.dom.persistent.DefaultDocumentSet)18 Sequence (org.exist.xquery.value.Sequence)18 MutableDocumentSet (org.exist.dom.persistent.MutableDocumentSet)16 NodeSet (org.exist.dom.persistent.NodeSet)14 DBBroker (org.exist.storage.DBBroker)14 BrokerPool (org.exist.storage.BrokerPool)13 CompiledXQuery (org.exist.xquery.CompiledXQuery)12 XQuery (org.exist.xquery.XQuery)12 IOException (java.io.IOException)9 Txn (org.exist.storage.txn.Txn)9 TransactionManager (org.exist.storage.txn.TransactionManager)8 Test (org.junit.Test)7 DocumentImpl (org.exist.dom.persistent.DocumentImpl)6 InputSource (org.xml.sax.InputSource)6 StringReader (java.io.StringReader)5 LuceneIndexWorker (org.exist.indexing.lucene.LuceneIndexWorker)5 XPathException (org.exist.xquery.XPathException)5 VirtualNodeSet (org.exist.dom.persistent.VirtualNodeSet)4