use of datawave.query.index.lookup.RangeStream in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method initializeRangeStream.
/**
* Initializes the range stream, whether it is configured to be a different class than the Default Range stream or not.
*
* @param config
* @param scannerFactory
* @param metadataHelper
* @return
*/
private RangeStream initializeRangeStream(ShardQueryConfiguration config, ScannerFactory scannerFactory, MetadataHelper metadataHelper) {
Class<? extends RangeStream> rstream;
try {
rstream = Class.forName(rangeStreamClass).asSubclass(RangeStream.class);
RangeStream stream = rstream.getConstructor(ShardQueryConfiguration.class, ScannerFactory.class, MetadataHelper.class).newInstance(config, scannerFactory, metadataHelper);
return stream.setUidIntersector(uidIntersector).setLimitScanners(limitScanners).setCreateUidsIteratorClass(createUidsIteratorClass);
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
throw new RuntimeException(e);
}
}
use of datawave.query.index.lookup.RangeStream in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method getQueryRanges.
/**
* Returns a Tuple2<Iterable<Range>,Boolean> whose elements represent the Ranges to use for querying the shard table and whether or not this is
* a "full-table-scan" query.
*
* @param scannerFactory
* @param metadataHelper
* @param config
* @param queryTree
* @return
* @throws DatawaveQueryException
*/
public Tuple2<CloseableIterable<QueryPlan>, Boolean> getQueryRanges(ScannerFactory scannerFactory, MetadataHelper metadataHelper, ShardQueryConfiguration config, JexlNode queryTree) throws DatawaveQueryException {
Preconditions.checkNotNull(queryTree);
boolean needsFullTable = false;
CloseableIterable<QueryPlan> ranges = null;
// if the query has already been reduced to false there is no reason to do more
if (QueryPruningVisitor.getState(queryTree) == QueryPruningVisitor.TruthState.FALSE) {
return new Tuple2<>(emptyCloseableIterator(), false);
}
// if we still have an unexecutable tree, then a full table scan is
// required
List<String> debugOutput = null;
if (log.isDebugEnabled()) {
debugOutput = new ArrayList<>(32);
}
STATE state = ExecutableDeterminationVisitor.getState(queryTree, config, metadataHelper, debugOutput);
if (log.isDebugEnabled()) {
logDebug(debugOutput, "ExecutableDeterminationVisitor at getQueryRanges:");
}
if (state != STATE.EXECUTABLE) {
if (state == STATE.ERROR) {
log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
throw new InvalidQueryException(qe);
}
log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
needsFullTable = true;
}
// scan, then lets try to compute ranges
if (!needsFullTable) {
// count the terms
int termCount = TermCountingVisitor.countTerms(queryTree);
if (termCount >= pushdownThreshold) {
if (log.isTraceEnabled()) {
log.trace("pushing down query because it has " + termCount + " when our max is " + pushdownThreshold);
}
config.setCollapseUids(true);
}
TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("DefaultQueryPlanner - Begin stream of ranges from inverted index");
RangeStream stream = initializeRangeStream(config, scannerFactory, metadataHelper);
ranges = stream.streamPlans(queryTree);
if (log.isTraceEnabled()) {
log.trace("query stream is " + stream.context());
}
// if a term threshold is exceeded and we cannot handle that, then
// throw unsupported
boolean thresholdExceeded = StreamContext.EXCEEDED_TERM_THRESHOLD.equals(stream.context());
if (thresholdExceeded && !config.canHandleExceededTermThreshold()) {
throw new UnsupportedOperationException(EXCEED_TERM_EXPANSION_ERROR);
}
if (StreamContext.UNINDEXED.equals(stream.context())) {
log.debug("Needs full table scan because of unindexed fields");
needsFullTable = true;
} else if (StreamContext.DELAYED_FIELD.equals(stream.context())) {
log.debug("Needs full table scan because query consists of only delayed expressions");
needsFullTable = true;
} else // force a full table scan
if (IvaratorRequiredVisitor.isIvaratorRequired(queryTree) && !config.canHandleExceededValueThreshold()) {
log.debug("Needs full table scan because we exceeded the value threshold and config.canHandleExceededValueThreshold() is false");
needsFullTable = true;
}
stopwatch.stop();
}
if (needsFullTable) {
if (config.getFullTableScanEnabled()) {
ranges = this.getFullScanRange(config, queryTree);
} else {
if (log.isTraceEnabled())
log.trace("Full table scans are not enabled, query will not be run");
QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
throw new FullTableScansDisallowedException(qe);
}
if (log.isTraceEnabled())
log.trace("Ranges are " + ranges);
}
return new Tuple2<>(ranges, needsFullTable);
}
Aggregations