Examples with CloseableIterable datawave.query.CloseableIterable used on opensource projects

Example 1 with CloseableIterable

use of datawave.query.CloseableIterable in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 2 with CloseableIterable

use of datawave.query.CloseableIterable in project datawave by NationalSecurityAgency.

the class BooleanChunkingQueryPlanner method process.

@Override
protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    final ArrayList<QueryData> data = Lists.newArrayList();
    ASTJexlScript queryTree = updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings);
    if (queryTree == null) {
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    String newQueryString = JexlStringBuildingVisitor.buildQuery(queryTree);
    if (StringUtils.isBlank(newQueryString)) {
        throw new DatawaveFatalQueryException("Query string after query modification was empty. Cannot run query.");
    } else if (log.isDebugEnabled()) {
        log.debug("Final query passed to QueryIterator:");
        log.debug(newQueryString);
    }
    queryData.setQuery(newQueryString);
    data.add(queryData);
    return new CloseableIterable<QueryData>() {

        @Override
        public Iterator<QueryData> iterator() {
            return data.iterator();
        }

        @Override
        public void close() throws IOException {
        }
    };
}