Examples with QueryStopwatch datawave.query.util.QueryStopwatch used on opensource projects

Example 1 with QueryStopwatch

use of datawave.query.util.QueryStopwatch in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
    if (!ShardQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) {
        throw new QueryException("Did not receive a ShardQueryConfiguration instance!!");
    }
    ShardQueryConfiguration config = (ShardQueryConfiguration) genericConfig;
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("ShardQueryLogic - Setup Query");
    // Ensure we have all of the information needed to run a query
    if (!config.canRunQuery()) {
        log.warn("The given query '" + config + "' could not be run, most likely due to not matching any records in the global index.");
        // Stub out an iterator to correctly present "no results"
        this.iterator = new Iterator<Map.Entry<Key, Value>>() {

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public Map.Entry<Key, Value> next() {
                return null;
            }

            @Override
            public void remove() {
                return;
            }
        };
        this.scanner = null;
        stopwatch.stop();
        log.info(getStopwatchHeader(config));
        List<String> timings = timers.summarizeAsList();
        for (String timing : timings) {
            log.info(timing);
        }
        return;
    }
    // Instantiate the scheduler for the queries
    this.scheduler = getScheduler(config, scannerFactory);
    this.scanner = null;
    this.iterator = this.scheduler.iterator();
    if (!config.isSortedUIDs()) {
        this.iterator = new DedupingIterator(this.iterator);
    }
    stopwatch.stop();
    log.info(getStopwatchHeader(config));
    List<String> timings = timers.summarizeAsList();
    for (String timing : timings) {
        log.info(timing);
    }
}

Example 2 with QueryStopwatch

use of datawave.query.util.QueryStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 3 with QueryStopwatch

use of datawave.query.util.QueryStopwatch in project datawave by NationalSecurityAgency.

the class BooleanChunkingQueryPlanner method updateQueryTree.

@Override
protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, QueryData queryData, Query settings) throws DatawaveQueryException {
    ASTJexlScript queryTree = super.updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings);
    if (queryTree == null) {
        return null;
    }
    final QueryStopwatch timer = config.getTimers();
    TraceStopwatch stopwatch = timer.newStartedStopwatch("BooleanChunkingQueryPlanner - Flatten duplicate nodes in query");
    queryTree = TreeFlatteningRebuildingVisitor.flatten(queryTree);
    if (log.isDebugEnabled() && queryTree != null) {
        log.debug("Query after flattening tree");
        for (String line : PrintingVisitor.formattedQueryStringList(queryTree)) {
            log.debug(line);
        }
        log.debug("");
    }
    stopwatch.stop();
    stopwatch = timer.newStartedStopwatch("BooleanChunkingQueryPlanner - Lift OR nodes in query tree");
    queryTree = BooleanOptimizationRebuildingVisitor.optimize(queryTree);
    if (log.isDebugEnabled() && queryTree != null) {
        log.debug("Query after optimizing boolean logic");
        for (String line : PrintingVisitor.formattedQueryStringList(queryTree)) {
            log.debug(line);
        }
        log.debug("");
    }
    stopwatch.stop();
    stopwatch = timer.newStartedStopwatch("BooleanChunkingQueryPlanner - Apply parenthesis to the new OR nodes");
    queryTree = TreeWrappingRebuildingVisitor.wrap(queryTree);
    if (log.isDebugEnabled() && queryTree != null) {
        log.debug("Query after wrapping tree");
        for (String line : PrintingVisitor.formattedQueryStringList(queryTree)) {
            log.debug(line);
        }
        log.debug("");
    }
    stopwatch.stop();
    return queryTree;
}

Example 4 with QueryStopwatch

use of datawave.query.util.QueryStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method validateQuerySize.

/**
 * NOT THREAD SAFE when called with timed=true
 *
 * @param lastOperation
 * @param queryTree
 * @param config
 * @param timed
 */
public static void validateQuerySize(String lastOperation, final JexlNode queryTree, ShardQueryConfiguration config, boolean timed) {
    TraceStopwatch stopwatch = null;
    if (timed) {
        final QueryStopwatch timers = config.getTimers();
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Validate against term and depth thresholds");
    }
    // check the query depth (up to config.getMaxDepthThreshold() + 1)
    int depth = DepthVisitor.getDepth(queryTree, config.getMaxDepthThreshold());
    if (depth > config.getMaxDepthThreshold()) {
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_THRESHOLD_EXCEEDED, MessageFormat.format("{0} > {1}, last operation: {2}", depth, config.getMaxDepthThreshold(), lastOperation));
        throw new DatawaveFatalQueryException(qe);
    }
    // count the terms
    int termCount = TermCountingVisitor.countTerms(queryTree);
    if (termCount > config.getMaxTermThreshold()) {
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_TERM_THRESHOLD_EXCEEDED, MessageFormat.format("{0} > {1}, last operation: {2}", termCount, config.getMaxTermThreshold(), lastOperation));
        throw new DatawaveFatalQueryException(qe);
    }
    if (timed) {
        stopwatch.stop();
    }
}

Example 5 with QueryStopwatch

use of datawave.query.util.QueryStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method updateQueryTree.

protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, QueryData queryData, Query settings) throws DatawaveQueryException {
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Parse query");
    config.setQueryTree(parseQueryAndValidatePattern(query, stopwatch));
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Query after initial parse:");
    }
    stopwatch.stop();
    Map<String, String> optionsMap = new HashMap<>();
    if (query.contains(QueryFunctions.QUERY_FUNCTION_NAMESPACE + ':')) {
        // only do the extra tree visit if the function is present
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - parse out queryOptions from options function");
        config.setQueryTree(QueryOptionsFromQueryVisitor.collect(config.getQueryTree(), optionsMap));
        if (!optionsMap.isEmpty()) {
            QueryOptionsSwitch.apply(optionsMap, config);
        }
        stopwatch.stop();
    }
    // groom the query so that any nodes with the literal on the left and the identifier on
    // the right will be re-ordered to simplify subsequent processing
    config.setQueryTree(timedInvertSwappedNodes(timers, config.getQueryTree()));
    config.setQueryTree(timedFixNotNullIntent(timers, config.getQueryTree()));
    config.setQueryTree(timedIncludeDateFilters(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, dateIndexHelper, settings));
    // note this must be called after we do the date adjustments per the query date type in addDateFilters
    timedCapDateRange(timers, config);
    // Find unmarked bounded ranges
    if (UnmarkedBoundedRangeDetectionVisitor.findUnmarkedBoundedRanges(config.getQueryTree())) {
        throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
    }
    if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
        config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
    } else {
        // look for the shards and days hint in the query settings
        // the shards and days hint cannot always be specified in the query string when using certain query parsers
        Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
        if (StringUtils.isNotBlank(parameter.getParameterValue())) {
            optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
            config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
        }
    }
    // extract #NO_EXPANSION function, if it exists
    config.setQueryTree(parseNoExpansionFields(timers, config.getQueryTree(), config));
    // flatten the tree
    config.setQueryTree(timedFlatten(timers, config.getQueryTree()));
    validateQuerySize("initial parse", config.getQueryTree(), config);
    config.setQueryTree(timedApplyRules(timers, config.getQueryTree(), config, metadataHelper, scannerFactory));
    config.setQueryTree(timedFixNegativeNumbers(timers, config.getQueryTree()));
    // Fix any query property markers that have multiple unwrapped sources.
    config.setQueryTree(timedFixQueryPropertyMarkers(timers, config.getQueryTree()));
    // Ensure that all ASTIdentifier nodes (field names) are upper-case to be consistent with what is enforced at ingest time
    config.setQueryTree(timedUpperCaseIdentifiers(timers, config.getQueryTree(), config, metadataHelper));
    config.setQueryTree(timedRewriteNegations(timers, config.getQueryTree()));
    QueryModel queryModel = loadQueryModel(config);
    config.setQueryTree(timedApplyQueryModel(timers, config.getQueryTree(), config, metadataHelper, queryModel));
    // Enforce unique terms within an AND or OR expression.
    if (config.getEnforceUniqueTermsWithinExpressions()) {
        config.setQueryTree(timedEnforceUniqueTermsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique AND'd terms within OR expressions.
    if (config.getEnforceUniqueConjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueConjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique OR'd terms within AND expressions.
    if (config.getEnforceUniqueDisjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueDisjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    Set<String> indexOnlyFields = loadIndexedFields(config);
    if (disableBoundedLookup) {
        // LT,GT or ER node, we should expand it
        if (BoundedRangeDetectionVisitor.mustExpandBoundedRange(config, metadataHelper, config.getQueryTree()))
            disableBoundedLookup = false;
    }
    if (!indexOnlyFields.isEmpty()) {
        config.setQueryTree(expandRegexFunctionNodes(config.getQueryTree(), config, metadataHelper, indexOnlyFields));
    }
    config.setQueryTree(processTree(config.getQueryTree(), config, settings, metadataHelper, scannerFactory, queryData, timers, queryModel));
    // ExpandCompositeTerms was here
    boolean containsIndexOnlyFields = false;
    if (!indexOnlyFields.isEmpty() && !disableBoundedLookup) {
        // Figure out if the query contained any index only terms so we know
        // if we have to force it down the field-index path with event-specific
        // ranges
        containsIndexOnlyFields = timedCheckForIndexOnlyFieldsInQuery(timers, "Check for Index-Only Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper, indexOnlyFields);
    }
    boolean containsComposites = timedCheckForCompositeFields(timers, "Check for Composite Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    boolean sortedUIDS = timedCheckForSortedUids(timers, "Check for Sorted UIDs", config.getQueryTree(), config);
    // check the query for any fields that are term frequencies
    // if any exist, populate the shard query config with these fields
    timedCheckForTokenizedFields(timers, "Check for term frequency (tokenized) fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query Final", config.getQueryTree()));
    }
    return config.getQueryTree();
}