Examples with FullTableScansDisallowedException datawave.query.exceptions.FullTableScansDisallowedException used on opensource projects

Example 1 with FullTableScansDisallowedException

use of datawave.query.exceptions.FullTableScansDisallowedException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 2 with FullTableScansDisallowedException

use of datawave.query.exceptions.FullTableScansDisallowedException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method getQueryRanges.

/**
 * Returns a Tuple2<Iterable<Range>,Boolean> whose elements represent the Ranges to use for querying the shard table and whether or not this is
 * a "full-table-scan" query.
 *
 * @param scannerFactory
 * @param metadataHelper
 * @param config
 * @param queryTree
 * @return
 * @throws DatawaveQueryException
 */
public Tuple2<CloseableIterable<QueryPlan>, Boolean> getQueryRanges(ScannerFactory scannerFactory, MetadataHelper metadataHelper, ShardQueryConfiguration config, JexlNode queryTree) throws DatawaveQueryException {
    Preconditions.checkNotNull(queryTree);
    boolean needsFullTable = false;
    CloseableIterable<QueryPlan> ranges = null;
    // if the query has already been reduced to false there is no reason to do more
    if (QueryPruningVisitor.getState(queryTree) == QueryPruningVisitor.TruthState.FALSE) {
        return new Tuple2<>(emptyCloseableIterator(), false);
    }
    // if we still have an unexecutable tree, then a full table scan is
    // required
    List<String> debugOutput = null;
    if (log.isDebugEnabled()) {
        debugOutput = new ArrayList<>(32);
    }
    STATE state = ExecutableDeterminationVisitor.getState(queryTree, config, metadataHelper, debugOutput);
    if (log.isDebugEnabled()) {
        logDebug(debugOutput, "ExecutableDeterminationVisitor at getQueryRanges:");
    }
    if (state != STATE.EXECUTABLE) {
        if (state == STATE.ERROR) {
            log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
            BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
            throw new InvalidQueryException(qe);
        }
        log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
        needsFullTable = true;
    }
    // scan, then lets try to compute ranges
    if (!needsFullTable) {
        // count the terms
        int termCount = TermCountingVisitor.countTerms(queryTree);
        if (termCount >= pushdownThreshold) {
            if (log.isTraceEnabled()) {
                log.trace("pushing down query because it has " + termCount + " when our max is " + pushdownThreshold);
            }
            config.setCollapseUids(true);
        }
        TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("DefaultQueryPlanner - Begin stream of ranges from inverted index");
        RangeStream stream = initializeRangeStream(config, scannerFactory, metadataHelper);
        ranges = stream.streamPlans(queryTree);
        if (log.isTraceEnabled()) {
            log.trace("query stream is " + stream.context());
        }
        // if a term threshold is exceeded and we cannot handle that, then
        // throw unsupported
        boolean thresholdExceeded = StreamContext.EXCEEDED_TERM_THRESHOLD.equals(stream.context());
        if (thresholdExceeded && !config.canHandleExceededTermThreshold()) {
            throw new UnsupportedOperationException(EXCEED_TERM_EXPANSION_ERROR);
        }
        if (StreamContext.UNINDEXED.equals(stream.context())) {
            log.debug("Needs full table scan because of unindexed fields");
            needsFullTable = true;
        } else if (StreamContext.DELAYED_FIELD.equals(stream.context())) {
            log.debug("Needs full table scan because query consists of only delayed expressions");
            needsFullTable = true;
        } else // force a full table scan
        if (IvaratorRequiredVisitor.isIvaratorRequired(queryTree) && !config.canHandleExceededValueThreshold()) {
            log.debug("Needs full table scan because we exceeded the value threshold and config.canHandleExceededValueThreshold() is false");
            needsFullTable = true;
        }
        stopwatch.stop();
    }
    if (needsFullTable) {
        if (config.getFullTableScanEnabled()) {
            ranges = this.getFullScanRange(config, queryTree);
        } else {
            if (log.isTraceEnabled())
                log.trace("Full table scans are not enabled, query will not be run");
            QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
        if (log.isTraceEnabled())
            log.trace("Ranges are " + ranges);
    }
    return new Tuple2<>(ranges, needsFullTable);
}

Example 3 with FullTableScansDisallowedException

use of datawave.query.exceptions.FullTableScansDisallowedException in project datawave by NationalSecurityAgency.

the class FacetedQueryPlanner method getQueryIterator.

@Override
public IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, Query settings, String queryString, Boolean isFullTable) throws DatawaveQueryException {
    if (isFullTable) {
        QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_DISALLOWED);
        throw new FullTableScansDisallowedException(qe);
    }
    IteratorSetting cfg = super.getQueryIterator(metadataHelper, config, settings, queryString, isFullTable);
    if (!usePrecomputedFacets)
        cfg.setIteratorClass(DynamicFacetIterator.class.getName());
    else {
        config.setShardTableName(facetedConfig.getFacetTableName());
        cfg.setIteratorClass(FacetedTableIterator.class.getName());
    }
    cfg.addOption(DynamicFacetIterator.FACETED_SEARCH_TYPE, facetedConfig.getType().toString());
    cfg.addOption(DynamicFacetIterator.FACETED_MINIMUM, Integer.toString(facetedConfig.getMinimumFacetCount()));
    cfg.addOption(DynamicFacetIterator.FACETED_SEARCH_FIELDS, Joiner.on(",").join(facetedConfig.getFacetedFields()));
    if (log.isTraceEnabled())
        log.trace("Configuration is " + facetedConfig);
    return cfg;
}

Example 4 with FullTableScansDisallowedException

use of datawave.query.exceptions.FullTableScansDisallowedException in project datawave by NationalSecurityAgency.

the class IndexQueryPlanner method getQueryIterator.

@Override
public IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, Query settings, String queryString, Boolean isFullTable) throws DatawaveQueryException {
    if (isFullTable) {
        QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_DISALLOWED);
        throw new FullTableScansDisallowedException(qe);
    }
    IteratorSetting cfg = super.getQueryIterator(metadataHelper, config, settings, queryString, isFullTable);
    if (null == cfg) {
        try {
            cfg = settingFuture.get();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }
    cfg.setIteratorClass(FieldIndexOnlyQueryIterator.class.getName());
    return cfg;
}