Examples with Indexable org.exist.storage.Indexable used on opensource projects

Example 1 with Indexable

use of org.exist.storage.Indexable in project exist by eXist-db.

the class GeneralComparison method preSelect.

public NodeSet preSelect(Sequence contextSequence, boolean useContext) throws XPathException {
    // the expression can be called multiple times, so we need to clear the previous preselectResult
    preselectResult = null;
    final long start = System.currentTimeMillis();
    final int indexType = Optimize.getQNameIndexType(context, contextSequence, contextQName);
    if (LOG.isTraceEnabled()) {
        LOG.trace("Using QName index on type {}", Type.getTypeName(indexType));
    }
    final Sequence rightSeq = getRight().eval(contextSequence);
    // into preselectResult
    if (rightSeq.getItemCount() > 1) {
        preselectResult = new NewArrayNodeSet();
    }
    // Iterate through each item in the right-hand sequence
    for (final SequenceIterator itRightSeq = Atomize.atomize(rightSeq).iterate(); itRightSeq.hasNext(); ) {
        // Get the index key
        Item key = itRightSeq.nextItem();
        // if key has truncation, convert it to string
        if (truncation != StringTruncationOperator.NONE) {
            if (!Type.subTypeOf(key.getType(), Type.STRING)) {
                LOG.info("Truncated key. Converted from {} to xs:string", Type.getTypeName(key.getType()));
                // truncation is only possible on strings
                key = key.convertTo(Type.STRING);
            }
        } else // TODO : use Type.isSubType() ??? -pb
        if (key.getType() != indexType) {
            // try to convert the key to the index type
            try {
                key = key.convertTo(indexType);
            } catch (final XPathException xpe) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Cannot convert key: {} to required index type: {}", Type.getTypeName(key.getType()), Type.getTypeName(indexType));
                }
                throw (new XPathException(this, "Cannot convert key to required index type"));
            }
        }
        // If key implements org.exist.storage.Indexable, we can use the index
        if (key instanceof Indexable) {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Using QName range index for key: {}", key.getStringValue());
            }
            NodeSet temp;
            final NodeSet contextSet = useContext ? contextSequence.toNodeSet() : null;
            final Collator collator = ((collationArg != null) ? getCollator(contextSequence) : null);
            if (truncation == StringTruncationOperator.NONE) {
                temp = context.getBroker().getValueIndex().find(context.getWatchDog(), relation, contextSequence.getDocumentSet(), contextSet, NodeSet.DESCENDANT, contextQName, (Indexable) key);
                hasUsedIndex = true;
            } else {
                try {
                    final String matchString = key.getStringValue();
                    final int matchType = getMatchType(truncation);
                    temp = context.getBroker().getValueIndex().match(context.getWatchDog(), contextSequence.getDocumentSet(), contextSet, NodeSet.DESCENDANT, matchString, contextQName, matchType, collator, truncation);
                    hasUsedIndex = true;
                } catch (final EXistException e) {
                    throw (new XPathException(this, "Error during index lookup: " + e.getMessage(), e));
                }
            }
            // else replace it.
            if (preselectResult == null) {
                preselectResult = temp;
            } else {
                preselectResult.addAll(temp);
            }
        }
    }
    if (context.getProfiler().traceFunctions()) {
        context.getProfiler().traceIndexUsage(context, PerformanceStats.RANGE_IDX_TYPE, this, PerformanceStats.OPTIMIZED_INDEX, System.currentTimeMillis() - start);
    }
    return ((preselectResult == null) ? NodeSet.EMPTY_SET : preselectResult);
}

Example 2 with Indexable

use of org.exist.storage.Indexable in project exist by eXist-db.

the class IndexKeys method eval.

/*
     * (non-Javadoc)
     * 
     * @see org.exist.xquery.BasicFunction#eval(org.exist.xquery.value.Sequence[],
     *      org.exist.xquery.value.Sequence)
     */
public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException {
    if (args[0].isEmpty()) {
        return Sequence.EMPTY_SEQUENCE;
    }
    NodeSet nodes = null;
    DocumentSet docs = null;
    Sequence qnames = null;
    if (isCalledAs("index-keys-by-qname")) {
        qnames = args[0];
        docs = contextSequence == null ? context.getStaticallyKnownDocuments() : contextSequence.getDocumentSet();
    } else {
        nodes = args[0].toNodeSet();
        docs = nodes.getDocumentSet();
    }
    final Sequence result = new ValueSequence();
    try (final FunctionReference ref = (FunctionReference) args[2].itemAt(0)) {
        int max = -1;
        if (args[3].hasOne()) {
            max = ((IntegerValue) args[3].itemAt(0)).getInt();
        }
        // if we have 5 arguments, query the user-specified index
        if (this.getArgumentCount() == 5) {
            final IndexWorker indexWorker = context.getBroker().getIndexController().getWorkerByIndexName(args[4].itemAt(0).getStringValue());
            // IndexWorker indexWorker = context.getBroker().getBrokerPool().getIndexManager().getIndexByName(args[4].itemAt(0).getStringValue()).getWorker();
            if (indexWorker == null) {
                throw new XPathException(this, "Unknown index: " + args[4].itemAt(0).getStringValue());
            }
            final Map<String, Object> hints = new HashMap<>();
            if (max != -1) {
                hints.put(IndexWorker.VALUE_COUNT, new IntegerValue(max));
            }
            if (indexWorker instanceof OrderedValuesIndex) {
                hints.put(OrderedValuesIndex.START_VALUE, args[1].getStringValue());
            } else {
                logger.warn("{} isn't an instance of org.exist.indexing.OrderedValuesIndex. Start value '{}' ignored.", indexWorker.getClass().getName(), args[1]);
            }
            if (qnames != null) {
                final List<QName> qnameList = new ArrayList<>(qnames.getItemCount());
                for (final SequenceIterator i = qnames.iterate(); i.hasNext(); ) {
                    final QNameValue qv = (QNameValue) i.nextItem();
                    qnameList.add(qv.getQName());
                }
                hints.put(QNamedKeysIndex.QNAMES_KEY, qnameList);
            }
            final Occurrences[] occur = indexWorker.scanIndex(context, docs, nodes, hints);
            // TODO : add an extra argument to pass the END_VALUE ?
            final int len = (max != -1 && occur.length > max ? max : occur.length);
            final Sequence[] params = new Sequence[2];
            ValueSequence data = new ValueSequence();
            for (int j = 0; j < len; j++) {
                params[0] = new StringValue(occur[j].getTerm().toString());
                data.add(new IntegerValue(occur[j].getOccurrences(), Type.UNSIGNED_INT));
                data.add(new IntegerValue(occur[j].getDocuments(), Type.UNSIGNED_INT));
                data.add(new IntegerValue(j + 1, Type.UNSIGNED_INT));
                params[1] = data;
                result.addAll(ref.evalFunction(Sequence.EMPTY_SEQUENCE, null, params));
                data.clear();
            }
        // no index specified: use the range index
        } else {
            final Indexable indexable = (Indexable) args[1].itemAt(0);
            ValueOccurrences[] occur = null;
            // First check for indexes defined on qname
            final QName[] allQNames = getDefinedIndexes(context.getBroker(), docs);
            if (allQNames.length > 0) {
                occur = context.getBroker().getValueIndex().scanIndexKeys(docs, nodes, allQNames, indexable);
            }
            // Also check if there's an index defined by path
            ValueOccurrences[] occur2 = context.getBroker().getValueIndex().scanIndexKeys(docs, nodes, indexable);
            // Merge the two results
            if (occur == null || occur.length == 0) {
                occur = occur2;
            } else {
                ValueOccurrences[] t = new ValueOccurrences[occur.length + occur2.length];
                System.arraycopy(occur, 0, t, 0, occur.length);
                System.arraycopy(occur2, 0, t, occur.length, occur2.length);
                occur = t;
            }
            final int len = (max != -1 && occur.length > max ? max : occur.length);
            final Sequence[] params = new Sequence[2];
            ValueSequence data = new ValueSequence();
            for (int j = 0; j < len; j++) {
                params[0] = occur[j].getValue();
                data.add(new IntegerValue(occur[j].getOccurrences(), Type.UNSIGNED_INT));
                data.add(new IntegerValue(occur[j].getDocuments(), Type.UNSIGNED_INT));
                data.add(new IntegerValue(j + 1, Type.UNSIGNED_INT));
                params[1] = data;
                result.addAll(ref.evalFunction(Sequence.EMPTY_SEQUENCE, null, params));
                data.clear();
            }
        }
    }
    logger.debug("Returning: {}", result.getItemCount());
    return result;
}

Example 3 with Indexable

use of org.exist.storage.Indexable in project exist by eXist-db.

the class IndexKeyOccurrences method eval.

public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException {
    if (context.getProfiler().isEnabled()) {
        context.getProfiler().start(this);
        context.getProfiler().message(this, Profiler.DEPENDENCIES, "DEPENDENCIES", Dependency.getDependenciesName(this.getDependencies()));
        if (contextSequence != null) {
            context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT SEQUENCE", contextSequence);
        }
    }
    Sequence result;
    if (args[0].isEmpty()) {
        result = Sequence.EMPTY_SEQUENCE;
    } else {
        final NodeSet nodes = args[0].toNodeSet();
        final DocumentSet docs = nodes.getDocumentSet();
        if (this.getArgumentCount() == 3) {
            final IndexWorker indexWorker = context.getBroker().getIndexController().getWorkerByIndexName(args[2].itemAt(0).getStringValue());
            // IndexWorker indexWorker = context.getBroker().getBrokerPool().getIndexManager().getIndexByName(args[2].itemAt(0).getStringValue()).getWorker();
            if (indexWorker == null) {
                throw new XPathException(this, "Unknown index: " + args[2].itemAt(0).getStringValue());
            }
            final Map<String, Object> hints = new HashMap<>();
            if (indexWorker instanceof OrderedValuesIndex) {
                hints.put(OrderedValuesIndex.START_VALUE, args[1]);
            } else {
                logger.warn("{} isn't an instance of org.exist.indexing.OrderedIndexWorker. Start value '{}' ignored.", indexWorker.getClass().getName(), args[1]);
            }
            final Occurrences[] occur = indexWorker.scanIndex(context, docs, nodes, hints);
            if (occur.length == 0) {
                result = Sequence.EMPTY_SEQUENCE;
            } else {
                result = new IntegerValue(occur[0].getOccurrences());
            }
        } else {
            ValueOccurrences[] occur = context.getBroker().getValueIndex().scanIndexKeys(docs, nodes, (Indexable) (args[1].itemAt(0)));
            if (occur.length == 0) {
                occur = context.getBroker().getValueIndex().scanIndexKeys(docs, nodes, null, (Indexable) (args[1].itemAt(0)));
            }
            if (occur.length == 0) {
                result = Sequence.EMPTY_SEQUENCE;
            } else {
                result = new IntegerValue(occur[0].getOccurrences());
            }
        }
    }
    if (context.getProfiler().isEnabled()) {
        context.getProfiler().end(this, "", result);
    }
    return result;
}

Example 4 with Indexable

use of org.exist.storage.Indexable in project exist by eXist-db.

the class GeneralComparison method quickNodeSetCompare.

/**
 * Optimized implementation: first checks if a range index is defined on the nodes in the left argument.
 * Otherwise, fall back to {@link #nodeSetCompare(NodeSet, Sequence)}.
 *
 * @param   contextSequence  DOCUMENT ME!
 *
 * @return  DOCUMENT ME!
 *
 * @throws  XPathException  DOCUMENT ME!
 */
protected Sequence quickNodeSetCompare(Sequence contextSequence) throws XPathException {
    if (context.getProfiler().isEnabled()) {
        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION CHOICE", "quickNodeSetCompare");
    }
    final long start = System.currentTimeMillis();
    // get the NodeSet on the left
    final Sequence leftSeq = getLeft().eval(contextSequence);
    if (!leftSeq.isPersistentSet()) {
        return (genericCompare(leftSeq, contextSequence, null));
    }
    final NodeSet nodes = leftSeq.isEmpty() ? NodeSet.EMPTY_SET : (NodeSet) leftSeq;
    // nothing on the left, so nothing to do
    if (!(nodes instanceof VirtualNodeSet) && nodes.isEmpty()) {
        // Well, we might discuss this one ;-)
        hasUsedIndex = true;
        return (Sequence.EMPTY_SEQUENCE);
    }
    // get the Sequence on the right
    final Sequence rightSeq = getRight().eval(contextSequence);
    // nothing on the right, so nothing to do
    if (rightSeq.isEmpty()) {
        // Well, we might discuss this one ;-)
        hasUsedIndex = true;
        return (Sequence.EMPTY_SEQUENCE);
    }
    // get the type of a possible index
    final int indexType = nodes.getIndexType();
    // remember that Type.ITEM means... no index ;-)
    if (indexType != Type.ITEM) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("found an index of type: {}", Type.getTypeName(indexType));
        }
        boolean indexScan = false;
        boolean indexMixed = false;
        QName myContextQName = contextQName;
        if (contextSequence != null) {
            final IndexFlags iflags = checkForQNameIndex(idxflags, context, contextSequence, myContextQName);
            boolean indexFound = false;
            if (!iflags.indexOnQName) {
                // if myContextQName != null and no index is defined on
                // myContextQName, we don't need to scan other QName indexes
                // and can just use the generic range index
                indexFound = myContextQName != null;
                if (iflags.partialIndexOnQName) {
                    indexMixed = true;
                } else {
                    // set myContextQName to null so the index lookup below is not
                    // restricted to that QName
                    myContextQName = null;
                }
            }
            if (!indexFound && (myContextQName == null)) {
                // we need to check them all
                if (iflags.hasIndexOnQNames) {
                    indexScan = true;
                }
            // else use range index defined on path by default
            }
        } else {
            return (nodeSetCompare(nodes, contextSequence));
        }
        // Get the documents from the node set
        final DocumentSet docs = nodes.getDocumentSet();
        // Holds the result
        NodeSet result = null;
        // Iterate through the right hand sequence
        for (final SequenceIterator itRightSeq = Atomize.atomize(rightSeq).iterate(); itRightSeq.hasNext(); ) {
            // Get the index key
            Item key = itRightSeq.nextItem();
            // if key has truncation, convert it to string
            if (truncation != StringTruncationOperator.NONE) {
                if (!Type.subTypeOf(key.getType(), Type.STRING)) {
                    LOG.info("Truncated key. Converted from {} to xs:string", Type.getTypeName(key.getType()));
                    // truncation is only possible on strings
                    key = key.convertTo(Type.STRING);
                }
            } else // TODO : use Type.isSubType() ??? -pb
            if (key.getType() != indexType) {
                // try to convert the key to the index type
                try {
                    key = key.convertTo(indexType);
                } catch (final XPathException xpe) {
                    // Could not convert the key to a suitable type for the index, fallback to nodeSetCompare()
                    if (context.getProfiler().isEnabled()) {
                        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (" + xpe.getMessage() + ")");
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Cannot convert key: {} to required index type: {}", Type.getTypeName(key.getType()), Type.getTypeName(indexType));
                    }
                    return (nodeSetCompare(nodes, contextSequence));
                }
            }
            // If key implements org.exist.storage.Indexable, we can use the index
            if (key instanceof Indexable) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Checking if range index can be used for key: {}", key.getStringValue());
                }
                final Collator collator = ((collationArg != null) ? getCollator(contextSequence) : null);
                if (Type.subTypeOf(key.getType(), indexType)) {
                    if (truncation == StringTruncationOperator.NONE) {
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Using range index for key: {}", key.getStringValue());
                        }
                        // key without truncation, find key
                        context.getProfiler().message(this, Profiler.OPTIMIZATIONS, "OPTIMIZATION", "Using value index '" + context.getBroker().getValueIndex().toString() + "' to find key '" + Type.getTypeName(key.getType()) + "(" + key.getStringValue() + ")'");
                        NodeSet ns;
                        if (indexScan) {
                            ns = context.getBroker().getValueIndex().findAll(context.getWatchDog(), relation, docs, nodes, NodeSet.ANCESTOR, (Indexable) key);
                        } else {
                            ns = context.getBroker().getValueIndex().find(context.getWatchDog(), relation, docs, nodes, NodeSet.ANCESTOR, myContextQName, (Indexable) key, indexMixed);
                        }
                        hasUsedIndex = true;
                        if (result == null) {
                            result = ns;
                        } else {
                            result = result.union(ns);
                        }
                    } else {
                        // key with truncation, match key
                        if (LOG.isTraceEnabled()) {
                            context.getProfiler().message(this, Profiler.OPTIMIZATIONS, "OPTIMIZATION", "Using value index '" + context.getBroker().getValueIndex().toString() + "' to match key '" + Type.getTypeName(key.getType()) + "(" + key.getStringValue() + ")'");
                        }
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("Using range index for key: {}", key.getStringValue());
                        }
                        try {
                            NodeSet ns;
                            final String matchString = key.getStringValue();
                            final int matchType = getMatchType(truncation);
                            if (indexScan) {
                                ns = context.getBroker().getValueIndex().matchAll(context.getWatchDog(), docs, nodes, NodeSet.ANCESTOR, matchString, matchType, 0, true, collator, truncation);
                            } else {
                                ns = context.getBroker().getValueIndex().match(context.getWatchDog(), docs, nodes, NodeSet.ANCESTOR, matchString, myContextQName, matchType, collator, truncation);
                            }
                            hasUsedIndex = true;
                            if (result == null) {
                                result = ns;
                            } else {
                                result = result.union(ns);
                            }
                        } catch (final EXistException e) {
                            throw (new XPathException(this, e));
                        }
                    }
                } else {
                    // our key does is not of the correct type
                    if (context.getProfiler().isEnabled()) {
                        context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (key is of type: " + Type.getTypeName(key.getType()) + ") whereas index is of type '" + Type.getTypeName(indexType) + "'");
                    }
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Cannot use range index: key is of type: {}) whereas index is of type '{}", Type.getTypeName(key.getType()), Type.getTypeName(indexType));
                    }
                    return (nodeSetCompare(nodes, contextSequence));
                }
            } else {
                // our key does not implement org.exist.storage.Indexable
                if (context.getProfiler().isEnabled()) {
                    context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "Falling back to nodeSetCompare (key is not an indexable type: " + key.getClass().getName());
                }
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Cannot use key which is of type '{}", key.getClass().getName());
                }
                return (nodeSetCompare(nodes, contextSequence));
            }
        }
        if (context.getProfiler().traceFunctions()) {
            context.getProfiler().traceIndexUsage(context, PerformanceStats.RANGE_IDX_TYPE, this, PerformanceStats.BASIC_INDEX, System.currentTimeMillis() - start);
        }
        return (result);
    } else {
        if (LOG.isTraceEnabled()) {
            LOG.trace("No suitable index found for key: {}", rightSeq.getStringValue());
        }
        // no range index defined on the nodes in this sequence, so fallback to nodeSetCompare
        if (context.getProfiler().isEnabled()) {
            context.getProfiler().message(this, Profiler.OPTIMIZATION_FLAGS, "OPTIMIZATION FALLBACK", "falling back to nodeSetCompare (no index available)");
        }
        return (nodeSetCompare(nodes, contextSequence));
    }
}