Examples with TraceStopwatch datawave.util.time.TraceStopwatch used on opensource projects

Example 26 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedAddDelayedPredicates.

protected ASTJexlScript timedAddDelayedPredicates(QueryStopwatch timers, String stage, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, Set<String> indexedFields, Set<String> indexOnlyFields, Set<String> nonEventFields, List<String> debugOutput) {
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    config.setQueryTree(script);
    if (log.isDebugEnabled()) {
        debugOutput.clear();
    }
    if (!ExecutableDeterminationVisitor.isExecutable(config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields, debugOutput, metadataHelper)) {
        config.setQueryTree((ASTJexlScript) PushdownUnexecutableNodesVisitor.pushdownPredicates(config.getQueryTree(), false, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper));
        if (log.isDebugEnabled()) {
            logDebug(debugOutput, "Executable state after expanding ranges and regex again:");
            logQuery(config.getQueryTree(), "Query after partially executable pushdown:");
        }
    }
    stopwatch.stop();
    return config.getQueryTree();
}

Example 27 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedRemoveDelayedPredicates.

protected ASTJexlScript timedRemoveDelayedPredicates(QueryStopwatch timers, String stage, ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, Set<String> indexedFields, Set<String> indexOnlyFields, Set<String> nonEventFields, Map<String, IndexLookup> indexLookupMap, ScannerFactory scannerFactory, MetadataHelper helper, List<String> debugOutput) throws TableNotFoundException {
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    config.setQueryTree((ASTJexlScript) PullupUnexecutableNodesVisitor.pullupDelayedPredicates(config.getQueryTree(), false, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper));
    if (log.isDebugEnabled()) {
        logDebug(debugOutput, "Executable state after expanding ranges:");
        logQuery(config.getQueryTree(), "Query after delayed pullup:");
    }
    boolean expandAllTerms = config.isExpandAllTerms();
    // set the expand all terms flag to avoid any more delayed
    // predicates based on cost...
    config.setExpandAllTerms(true);
    // Check if there is any regex to expand after pulling up delayed predicates.
    NodeTypeCount nodeCount = NodeTypeCountVisitor.countNodes(config.getQueryTree());
    if (nodeCount.hasAny(ASTNRNode.class, ASTERNode.class)) {
        config.setQueryTree(RegexIndexExpansionVisitor.expandRegex(config, scannerFactory, helper, indexLookupMap, config.getQueryTree()));
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Query after expanding regex again:");
        }
    }
    // Check if there are any bounded ranges to expand.
    if (nodeCount.isPresent(BoundedRange.class)) {
        try {
            config.setQueryTree(BoundedRangeIndexExpansionVisitor.expandBoundedRanges(config, scannerFactory, metadataHelper, config.getQueryTree()));
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
            throw new DatawaveFatalQueryException(qe);
        }
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Query after expanding ranges again:");
        }
    }
    if (reduceQuery) {
        // only show pruned sections of the tree's via assignments if debug to reduce runtime when possible
        config.setQueryTree((ASTJexlScript) QueryPruningVisitor.reduce(config.getQueryTree(), showReducedQueryPrune));
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Query after range expansion reduction again:");
        }
    }
    // Check if there are functions that can be pushed into exceeded value ranges.
    if (nodeCount.hasAll(ASTFunctionNode.class, ExceededValueThresholdMarkerJexlNode.class)) {
        config.setQueryTree(PushFunctionsIntoExceededValueRanges.pushFunctions(config.getQueryTree(), metadataHelper, config.getDatatypeFilter()));
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Query after expanding pushing functions into exceeded value ranges again:");
        }
    }
    // Reset the original expandAllTerms value.
    config.setExpandAllTerms(expandAllTerms);
    stopwatch.stop();
    return config.getQueryTree();
}

Example 28 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedFetchDatatypes.

protected void timedFetchDatatypes(QueryStopwatch timers, String stage, final ASTJexlScript script, ShardQueryConfiguration config) throws DatawaveQueryException {
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    try {
        Multimap<String, Type<?>> fieldToDatatypeMap = null;
        if (cacheDataTypes) {
            fieldToDatatypeMap = dataTypeMap.getIfPresent(String.valueOf(config.getDatatypeFilter().hashCode()));
        }
        if (null != fieldToDatatypeMap) {
            Set<String> indexedFields = Sets.newHashSet();
            Set<String> reverseIndexedFields = Sets.newHashSet();
            Set<String> normalizedFields = Sets.newHashSet();
            loadDataTypeMetadata(fieldToDatatypeMap, indexedFields, reverseIndexedFields, normalizedFields, false);
            if (null == queryFieldsAsDataTypeMap) {
                queryFieldsAsDataTypeMap = HashMultimap.create(Multimaps.filterKeys(fieldToDatatypeMap, input -> !cachedNormalizedFields.contains(input)));
            }
            if (null == normalizedFieldAsDataTypeMap) {
                normalizedFieldAsDataTypeMap = HashMultimap.create(Multimaps.filterKeys(fieldToDatatypeMap, input -> cachedNormalizedFields.contains(input)));
            }
            setCachedFields(indexedFields, reverseIndexedFields, queryFieldsAsDataTypeMap, normalizedFieldAsDataTypeMap, config);
        } else {
            fieldToDatatypeMap = configureIndexedAndNormalizedFields(metadataHelper, config, script);
            if (cacheDataTypes) {
                loadDataTypeMetadata(null, null, null, null, true);
                dataTypeMap.put(String.valueOf(config.getDatatypeFilter().hashCode()), metadataHelper.getFieldsToDatatypes(config.getDatatypeFilter()));
            }
        }
    } catch (InstantiationException | IllegalAccessException | AccumuloException | AccumuloSecurityException | TableNotFoundException | ExecutionException e) {
        throw new DatawaveFatalQueryException(e);
    } finally {
        stopwatch.stop();
    }
}

Example 29 with TraceStopwatch

use of datawave.util.time.TraceStopwatch in project datawave by NationalSecurityAgency.

the class TimedVisitorManager method timedVisit.

/**
 * Wrap visitor execution with a timer and debug msg
 *
 * @param timers
 *            a {@link QueryStopwatch}
 * @param stageName
 *            the name for this operation, for example "Fix Not Null Intent" or "Expand Regex Nodes"
 * @param visitorManager
 *            an interface that allows us to pass a lambda operation {@link VisitorManager}
 * @return the query tree, a {@link ASTJexlScript}
 * @throws DatawaveQueryException
 *             if something goes wrong
 */
public ASTJexlScript timedVisit(QueryStopwatch timers, String stageName, VisitorManager visitorManager) throws DatawaveQueryException {
    ASTJexlScript script;
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stageName);
    try {
        script = visitorManager.apply();
        if (isDebugEnabled) {
            logQuery(script, "Query after visit: " + stageName);
        }
        if (validateAst) {
            try {
                ASTValidator.isValid(script, stageName);
            } catch (InvalidQueryTreeException e) {
                throw new DatawaveQueryException(e);
            }
        }
    } finally {
        stopwatch.stop();
    }
    return script;
}