Examples with TraceStopwatch datawave.util.time.TraceStopwatch used on opensource projects

Example 21 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method processTree.

protected ASTJexlScript processTree(final ASTJexlScript originalQueryTree, ShardQueryConfiguration config, Query settings, MetadataHelper metadataHelper, ScannerFactory scannerFactory, QueryData queryData, QueryStopwatch timers, QueryModel queryModel) throws DatawaveQueryException {
    config.setQueryTree(originalQueryTree);
    TraceStopwatch stopwatch = null;
    if (!disableWhindexFieldMappings) {
        // apply the value-specific field mappings for GeoWave functions
        config.setQueryTree(timedApplyWhindexFieldMappings(timers, config.getQueryTree(), config, metadataHelper, settings));
    }
    if (!disableExpandIndexFunction) {
        // expand the index queries for the functions
        config.setQueryTree(timedExpandIndexQueriesForFunctions(timers, config.getQueryTree(), config, metadataHelper));
    }
    // apply the node transform rules
    // running it here before any unfielded expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Unfielded Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // left as a regex
    if (!disableAnyFieldLookup) {
        config.setQueryTree(timedExpandAnyFieldRegexNodes(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, settings.getQuery()));
    }
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query After ANYFIELD Expansions", config.getQueryTree()));
    }
    if (!disableTestNonExistentFields) {
        timedTestForNonExistentFields(timers, config.getQueryTree(), config, metadataHelper, queryModel, settings);
    }
    // apply the node transform rules
    // running it here before any regex or range expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Regex/Range Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    timedFetchDatatypes(timers, "Fetch Required Datatypes", config.getQueryTree(), config);
    config.setQueryTree(timedFixUnindexedNumerics(timers, config.getQueryTree(), config));
    config.setQueryTree(timedExpandMultiNormalizedTerms(timers, config.getQueryTree(), config, metadataHelper));
    // if we have any index holes, then mark em
    if (!config.getIndexHoles().isEmpty()) {
        config.setQueryTree(timedMarkIndexHoles(timers, config.getQueryTree(), config, metadataHelper));
    }
    // lets precompute the indexed fields and index only fields for the specific datatype if needed below
    Set<String> indexedFields = null;
    Set<String> indexOnlyFields = null;
    Set<String> nonEventFields = null;
    if (config.getMinSelectivity() > 0 || !disableBoundedLookup) {
        try {
            indexedFields = metadataHelper.getIndexedFields(config.getDatatypeFilter());
            indexOnlyFields = metadataHelper.getIndexOnlyFields(config.getDatatypeFilter());
            nonEventFields = metadataHelper.getNonEventFields(config.getDatatypeFilter());
        } catch (TableNotFoundException te) {
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, te);
            throw new DatawaveFatalQueryException(qe);
        }
    }
    // apply the node transform rules
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Pushdown/Pullup Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // push down terms that are over the min selectivity
    if (config.getMinSelectivity() > 0) {
        config.setQueryTree(timedPushdownLowSelectiveTerms(timers, config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields));
    }
    config.setQueryTree(timedForceFieldToFieldComparison(timers, config.getQueryTree()));
    if (!disableCompositeFields) {
        config.setQueryTree(timedExpandCompositeFields(timers, config.getQueryTree(), config));
    }
    if (!disableBoundedLookup) {
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Expand bounded query ranges (total)");
        // Expand any bounded ranges into a conjunction of discrete terms
        try {
            Map<String, IndexLookup> indexLookupMap = new HashMap<>();
            // Check if there is any regex to expand.
            NodeTypeCount nodeCount = NodeTypeCountVisitor.countNodes(config.getQueryTree());
            if (nodeCount.hasAny(ASTNRNode.class, ASTERNode.class)) {
                config.setQueryTree(timedExpandRegex(timers, "Expand Regex", config.getQueryTree(), config, metadataHelper, scannerFactory, indexLookupMap));
            }
            // Check if there are any bounded ranges to expand.
            if (nodeCount.isPresent(BoundedRange.class)) {
                config.setQueryTree(timedExpandRanges(timers, "Expand Ranges", config.getQueryTree(), config, metadataHelper, scannerFactory));
            }
            // NOTE: GeoWavePruningVisitor should run before QueryPruningVisitor. If it runs after, there is a chance
            // that GeoWavePruningVisitor will prune all of the remaining indexed terms, which would leave a GeoWave
            // function without any indexed terms or ranges, which should evaluate to false. That case won't be handled
            // properly if we run GeoWavePruningVisitor after QueryPruningVisitor.
            config.setQueryTree(timedPruneGeoWaveTerms(timers, config.getQueryTree(), metadataHelper));
            if (reduceQuery) {
                config.setQueryTree(timedReduce(timers, "Reduce Query After Range Expansion", config.getQueryTree()));
            }
            // Check if there are functions that can be pushed into exceeded value ranges.
            if (nodeCount.hasAll(ASTFunctionNode.class, ExceededValueThresholdMarkerJexlNode.class)) {
                config.setQueryTree(timedPushFunctions(timers, config.getQueryTree(), config, metadataHelper));
            }
            if (executableExpansion) {
                config.setQueryTree(timedExecutableExpansion(timers, config.getQueryTree(), config, metadataHelper));
            }
            List<String> debugOutput = null;
            if (log.isDebugEnabled()) {
                debugOutput = new ArrayList<>(32);
            }
            // the terms to be delayed.
            if (!ExecutableDeterminationVisitor.isExecutable(config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields, debugOutput, metadataHelper)) {
                // if we now have an unexecutable tree because of delayed
                // predicates, then remove delayed predicates as needed and
                // reexpand
                config.setQueryTree(timedRemoveDelayedPredicates(timers, "Remove Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, indexLookupMap, scannerFactory, metadataHelper, debugOutput));
            }
            // if we now have an unexecutable tree because of missing
            // delayed predicates, then add delayed predicates where
            // possible
            config.setQueryTree(timedAddDelayedPredicates(timers, "Add Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, debugOutput));
        } catch (TableNotFoundException e) {
            stopwatch.stop();
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
            throw new DatawaveFatalQueryException(qe);
        } catch (CannotExpandUnfieldedTermFatalException e) {
            if (null != e.getCause() && e.getCause() instanceof DoNotPerformOptimizedQueryException)
                throw (DoNotPerformOptimizedQueryException) e.getCause();
            QueryException qe = new QueryException(DatawaveErrorCode.INDETERMINATE_INDEX_STATUS, e);
            throw new DatawaveFatalQueryException(qe);
        }
        stopwatch.stop();
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Bounded range and regex conversion has been disabled");
        }
    }
    return config.getQueryTree();
}

Example 22 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method validateQuerySize.

/**
 * NOT THREAD SAFE when called with timed=true
 *
 * @param lastOperation
 * @param queryTree
 * @param config
 * @param timed
 */
public static void validateQuerySize(String lastOperation, final JexlNode queryTree, ShardQueryConfiguration config, boolean timed) {
    TraceStopwatch stopwatch = null;
    if (timed) {
        final QueryStopwatch timers = config.getTimers();
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Validate against term and depth thresholds");
    }
    // check the query depth (up to config.getMaxDepthThreshold() + 1)
    int depth = DepthVisitor.getDepth(queryTree, config.getMaxDepthThreshold());
    if (depth > config.getMaxDepthThreshold()) {
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_THRESHOLD_EXCEEDED, MessageFormat.format("{0} > {1}, last operation: {2}", depth, config.getMaxDepthThreshold(), lastOperation));
        throw new DatawaveFatalQueryException(qe);
    }
    // count the terms
    int termCount = TermCountingVisitor.countTerms(queryTree);
    if (termCount > config.getMaxTermThreshold()) {
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_TERM_THRESHOLD_EXCEEDED, MessageFormat.format("{0} > {1}, last operation: {2}", termCount, config.getMaxTermThreshold(), lastOperation));
        throw new DatawaveFatalQueryException(qe);
    }
    if (timed) {
        stopwatch.stop();
    }
}

Example 23 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedCheckForCompositeFields.

protected boolean timedCheckForCompositeFields(QueryStopwatch timers, String stage, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper) {
    boolean containsComposites = false;
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    Multimap<String, String> compositeFields;
    try {
        compositeFields = metadataHelper.getCompositeToFieldMap(config.getDatatypeFilter());
    } catch (TableNotFoundException e) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.COMPOSITES_RETRIEVAL_ERROR, e);
        throw new DatawaveFatalQueryException(qe);
    }
    // ranges
    if (!compositeFields.isEmpty()) {
        boolean functionsEnabled = config.isCompositeFilterFunctionsEnabled();
        containsComposites = !SetMembershipVisitor.getMembers(compositeFields.keySet(), config, metadataHelper, dateIndexHelper, config.getQueryTree(), functionsEnabled).isEmpty();
    }
    // Print the nice log message
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Computed that the query " + (containsComposites ? " contains " : " does not contain any ") + " composite field(s)");
    }
    config.setContainsCompositeTerms(containsComposites);
    stopwatch.stop();
    return containsComposites;
}

Example 24 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method updateQueryTree.

protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, QueryData queryData, Query settings) throws DatawaveQueryException {
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Parse query");
    config.setQueryTree(parseQueryAndValidatePattern(query, stopwatch));
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Query after initial parse:");
    }
    stopwatch.stop();
    Map<String, String> optionsMap = new HashMap<>();
    if (query.contains(QueryFunctions.QUERY_FUNCTION_NAMESPACE + ':')) {
        // only do the extra tree visit if the function is present
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - parse out queryOptions from options function");
        config.setQueryTree(QueryOptionsFromQueryVisitor.collect(config.getQueryTree(), optionsMap));
        if (!optionsMap.isEmpty()) {
            QueryOptionsSwitch.apply(optionsMap, config);
        }
        stopwatch.stop();
    }
    // groom the query so that any nodes with the literal on the left and the identifier on
    // the right will be re-ordered to simplify subsequent processing
    config.setQueryTree(timedInvertSwappedNodes(timers, config.getQueryTree()));
    config.setQueryTree(timedFixNotNullIntent(timers, config.getQueryTree()));
    config.setQueryTree(timedIncludeDateFilters(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, dateIndexHelper, settings));
    // note this must be called after we do the date adjustments per the query date type in addDateFilters
    timedCapDateRange(timers, config);
    // Find unmarked bounded ranges
    if (UnmarkedBoundedRangeDetectionVisitor.findUnmarkedBoundedRanges(config.getQueryTree())) {
        throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
    }
    if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
        config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
    } else {
        // look for the shards and days hint in the query settings
        // the shards and days hint cannot always be specified in the query string when using certain query parsers
        Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
        if (StringUtils.isNotBlank(parameter.getParameterValue())) {
            optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
            config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
        }
    }
    // extract #NO_EXPANSION function, if it exists
    config.setQueryTree(parseNoExpansionFields(timers, config.getQueryTree(), config));
    // flatten the tree
    config.setQueryTree(timedFlatten(timers, config.getQueryTree()));
    validateQuerySize("initial parse", config.getQueryTree(), config);
    config.setQueryTree(timedApplyRules(timers, config.getQueryTree(), config, metadataHelper, scannerFactory));
    config.setQueryTree(timedFixNegativeNumbers(timers, config.getQueryTree()));
    // Fix any query property markers that have multiple unwrapped sources.
    config.setQueryTree(timedFixQueryPropertyMarkers(timers, config.getQueryTree()));
    // Ensure that all ASTIdentifier nodes (field names) are upper-case to be consistent with what is enforced at ingest time
    config.setQueryTree(timedUpperCaseIdentifiers(timers, config.getQueryTree(), config, metadataHelper));
    config.setQueryTree(timedRewriteNegations(timers, config.getQueryTree()));
    QueryModel queryModel = loadQueryModel(config);
    config.setQueryTree(timedApplyQueryModel(timers, config.getQueryTree(), config, metadataHelper, queryModel));
    // Enforce unique terms within an AND or OR expression.
    if (config.getEnforceUniqueTermsWithinExpressions()) {
        config.setQueryTree(timedEnforceUniqueTermsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique AND'd terms within OR expressions.
    if (config.getEnforceUniqueConjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueConjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique OR'd terms within AND expressions.
    if (config.getEnforceUniqueDisjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueDisjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    Set<String> indexOnlyFields = loadIndexedFields(config);
    if (disableBoundedLookup) {
        // LT,GT or ER node, we should expand it
        if (BoundedRangeDetectionVisitor.mustExpandBoundedRange(config, metadataHelper, config.getQueryTree()))
            disableBoundedLookup = false;
    }
    if (!indexOnlyFields.isEmpty()) {
        config.setQueryTree(expandRegexFunctionNodes(config.getQueryTree(), config, metadataHelper, indexOnlyFields));
    }
    config.setQueryTree(processTree(config.getQueryTree(), config, settings, metadataHelper, scannerFactory, queryData, timers, queryModel));
    // ExpandCompositeTerms was here
    boolean containsIndexOnlyFields = false;
    if (!indexOnlyFields.isEmpty() && !disableBoundedLookup) {
        // Figure out if the query contained any index only terms so we know
        // if we have to force it down the field-index path with event-specific
        // ranges
        containsIndexOnlyFields = timedCheckForIndexOnlyFieldsInQuery(timers, "Check for Index-Only Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper, indexOnlyFields);
    }
    boolean containsComposites = timedCheckForCompositeFields(timers, "Check for Composite Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    boolean sortedUIDS = timedCheckForSortedUids(timers, "Check for Sorted UIDs", config.getQueryTree(), config);
    // check the query for any fields that are term frequencies
    // if any exist, populate the shard query config with these fields
    timedCheckForTokenizedFields(timers, "Check for term frequency (tokenized) fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query Final", config.getQueryTree()));
    }
    return config.getQueryTree();
}

Example 25 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedFixQueryPropertyMarkers.

protected ASTJexlScript timedFixQueryPropertyMarkers(QueryStopwatch timers, ASTJexlScript script) throws DatawaveQueryException {
    // Fix query property markers with multiple sources.
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - fix query property markers with multiple sources");
    try {
        script = QueryPropertyMarkerSourceConsolidator.consolidate(script);
    } catch (Exception e) {
        throw new DatawaveQueryException("Failed to fix query property markers with multiple sources", e);
    }
    if (log.isDebugEnabled()) {
        logQuery(script, "Query after fixing query property markers with multiple sources");
    }
    stopwatch.stop();
    return script;
}