Examples with QueryData datawave.webservice.query.configuration.QueryData used on opensource projects

Example 1 with QueryData

use of datawave.webservice.query.configuration.QueryData in project datawave by NationalSecurityAgency.

the class DefaultExtendedEdgeQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    EdgeExtendedSummaryConfiguration localConf = (EdgeExtendedSummaryConfiguration) configuration;
    config = (EdgeExtendedSummaryConfiguration) configuration;
    prefilterValues = null;
    EdgeExtendedSummaryConfiguration.dateType dateFilterType = localConf.getDateRangeType();
    if (log.isTraceEnabled()) {
        log.trace("Performing edge table query: " + config.getQueryString());
    }
    // TODO check to see if overriding I/O necessary
    if (allowOverrideIO && localConf.isOverRideInput()) {
        this.summaryInputType = localConf.isSummaryInputType();
    }
    if (allowOverrideIO && localConf.isOverRideOutput()) {
        this.summaryOutputType = localConf.isAggregateResults();
    }
    boolean includeStats = localConf.includeStats();
    String queryString = config.getQueryString();
    MetadataHelper metadataHelper = super.prepareMetadataHelper(config.getConnector(), config.getModelTableName(), config.getAuthorizations());
    loadQueryModel(metadataHelper, config);
    // list of source values...no field names to translate
    if (this.summaryInputType == false) {
        queryString = applyQueryModel(queryString);
    }
    // set the modified queryString back into the config, for easy access
    config.setQueryString(queryString);
    String normalizedQuery = "";
    String statsNormalizedQuery = "";
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    VisitationContext context = null;
    if (this.summaryInputType == false) {
        try {
            context = normalizeJexlQuery(queryString, false);
            normalizedQuery = context.getNormalizedQuery().toString();
            statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
            if (log.isTraceEnabled()) {
                log.trace("Jexl after normalizing both vertices: " + normalizedQuery);
            }
        } catch (JexlException ex) {
            try {
                log.error("Error parsing user query.", ex);
            } catch (Exception ex2) {
                log.error("Exception thrown by logger (???)");
            }
        }
    }
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterScanLimit, dateFilterType));
    if (!normalizedQuery.equals("")) {
        if (log.isTraceEnabled()) {
            log.trace("Query being sent to the filter iterator: " + normalizedQuery);
        }
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_" + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        addIterator(qData, edgeIteratorSetting);
    }
    if (log.isTraceEnabled()) {
        log.trace("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    }
    BatchScanner scanner = createBatchScanner(config);
    if (log.isTraceEnabled()) {
        log.trace("Using the following ranges: " + qData.getRanges());
    }
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
    }
    scanner.setRanges(qData.getRanges());
    addCustomFilters(qData, currentIteratorPriority);
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    this.scanner = scanner;
    iterator = scanner.iterator();
}

Example 2 with QueryData

use of datawave.webservice.query.configuration.QueryData in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 3 with QueryData

use of datawave.webservice.query.configuration.QueryData in project datawave by NationalSecurityAgency.

the class CompositeIndexTest method getQueryRanges.

private List<QueryData> getQueryRanges(String queryString, boolean useIvarator) throws Exception {
    ShardQueryLogic logic = getShardQueryLogic(useIvarator);
    Iterator iter = getQueryRangesIterator(queryString, logic);
    List<QueryData> queryData = new ArrayList<>();
    while (iter.hasNext()) queryData.add((QueryData) iter.next());
    return queryData;
}

Example 4 with QueryData

use of datawave.webservice.query.configuration.QueryData in project datawave by NationalSecurityAgency.

the class MultiValueCompositeIndexTest method getQueryRanges.

private List<QueryData> getQueryRanges(String queryString, boolean useIvarator) throws Exception {
    ShardQueryLogic logic = getShardQueryLogic(useIvarator);
    Iterator iter = getQueryRangesIterator(queryString, logic);
    List<QueryData> queryData = new ArrayList<>();
    while (iter.hasNext()) queryData.add((QueryData) iter.next());
    return queryData;
}

Example 5 with QueryData

use of datawave.webservice.query.configuration.QueryData in project datawave by NationalSecurityAgency.

the class ThreadedRangeBundlerTest method whenInstantiatingViaModifiedBuilder_thenSpecifiedValuesAreSet.

@Test
public void whenInstantiatingViaModifiedBuilder_thenSpecifiedValuesAreSet() {
    QueryData original = mock(QueryData.class);
    CloseableIterable<QueryPlan> ranges = mock(CloseableIterable.class);
    Query settings = mock(Query.class);
    ASTJexlScript queryTree = mock(ASTJexlScript.class);
    Collection<Comparator<QueryPlan>> queryPlanComparators = mock(Collection.class);
    // @formatter:off
    ThreadedRangeBundler bundler = ThreadedRangeBundler.builder().setOriginal(original).setRanges(ranges).setMaxRanges(100).setSettings(settings).setQueryTree(queryTree).setDocSpecificLimitOverride(true).setDocsToCombine(10).setMaxRangeWaitMillis(1).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(1).setRangeBufferTimeoutMillis(10).setRangeBufferPollMillis(5).build();
    // @formatter:on
    assertEquals(original, bundler.getOriginal());
    assertEquals(ranges, bundler.getRanges());
    assertEquals(100L, bundler.getMaxRanges());
    assertEquals(settings, bundler.getSettings());
    assertEquals(queryTree, bundler.getQueryTree());
    assertTrue(bundler.isDocSpecificLimitOverride());
    assertEquals(10, bundler.getDocsToCombine());
    assertEquals(queryPlanComparators, bundler.getQueryPlanComparators());
    assertEquals(1, bundler.getNumRangesToBuffer());
    assertEquals(10L, bundler.getRangeBufferTimeoutMillis());
    assertEquals(5L, bundler.getRangeBufferPollMillis());
    assertEquals(1L, bundler.getMaxRangeWaitMillis());
}