Examples with DoNotPerformOptimizedQueryException datawave.query.exceptions.DoNotPerformOptimizedQueryException used on opensource projects

Example 1 with DoNotPerformOptimizedQueryException

use of datawave.query.exceptions.DoNotPerformOptimizedQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method processTree.

protected ASTJexlScript processTree(final ASTJexlScript originalQueryTree, ShardQueryConfiguration config, Query settings, MetadataHelper metadataHelper, ScannerFactory scannerFactory, QueryData queryData, QueryStopwatch timers, QueryModel queryModel) throws DatawaveQueryException {
    config.setQueryTree(originalQueryTree);
    TraceStopwatch stopwatch = null;
    if (!disableWhindexFieldMappings) {
        // apply the value-specific field mappings for GeoWave functions
        config.setQueryTree(timedApplyWhindexFieldMappings(timers, config.getQueryTree(), config, metadataHelper, settings));
    }
    if (!disableExpandIndexFunction) {
        // expand the index queries for the functions
        config.setQueryTree(timedExpandIndexQueriesForFunctions(timers, config.getQueryTree(), config, metadataHelper));
    }
    // apply the node transform rules
    // running it here before any unfielded expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Unfielded Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // left as a regex
    if (!disableAnyFieldLookup) {
        config.setQueryTree(timedExpandAnyFieldRegexNodes(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, settings.getQuery()));
    }
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query After ANYFIELD Expansions", config.getQueryTree()));
    }
    if (!disableTestNonExistentFields) {
        timedTestForNonExistentFields(timers, config.getQueryTree(), config, metadataHelper, queryModel, settings);
    }
    // apply the node transform rules
    // running it here before any regex or range expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Regex/Range Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    timedFetchDatatypes(timers, "Fetch Required Datatypes", config.getQueryTree(), config);
    config.setQueryTree(timedFixUnindexedNumerics(timers, config.getQueryTree(), config));
    config.setQueryTree(timedExpandMultiNormalizedTerms(timers, config.getQueryTree(), config, metadataHelper));
    // if we have any index holes, then mark em
    if (!config.getIndexHoles().isEmpty()) {
        config.setQueryTree(timedMarkIndexHoles(timers, config.getQueryTree(), config, metadataHelper));
    }
    // lets precompute the indexed fields and index only fields for the specific datatype if needed below
    Set<String> indexedFields = null;
    Set<String> indexOnlyFields = null;
    Set<String> nonEventFields = null;
    if (config.getMinSelectivity() > 0 || !disableBoundedLookup) {
        try {
            indexedFields = metadataHelper.getIndexedFields(config.getDatatypeFilter());
            indexOnlyFields = metadataHelper.getIndexOnlyFields(config.getDatatypeFilter());
            nonEventFields = metadataHelper.getNonEventFields(config.getDatatypeFilter());
        } catch (TableNotFoundException te) {
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, te);
            throw new DatawaveFatalQueryException(qe);
        }
    }
    // apply the node transform rules
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Pushdown/Pullup Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // push down terms that are over the min selectivity
    if (config.getMinSelectivity() > 0) {
        config.setQueryTree(timedPushdownLowSelectiveTerms(timers, config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields));
    }
    config.setQueryTree(timedForceFieldToFieldComparison(timers, config.getQueryTree()));
    if (!disableCompositeFields) {
        config.setQueryTree(timedExpandCompositeFields(timers, config.getQueryTree(), config));
    }
    if (!disableBoundedLookup) {
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Expand bounded query ranges (total)");
        // Expand any bounded ranges into a conjunction of discrete terms
        try {
            Map<String, IndexLookup> indexLookupMap = new HashMap<>();
            // Check if there is any regex to expand.
            NodeTypeCount nodeCount = NodeTypeCountVisitor.countNodes(config.getQueryTree());
            if (nodeCount.hasAny(ASTNRNode.class, ASTERNode.class)) {
                config.setQueryTree(timedExpandRegex(timers, "Expand Regex", config.getQueryTree(), config, metadataHelper, scannerFactory, indexLookupMap));
            }
            // Check if there are any bounded ranges to expand.
            if (nodeCount.isPresent(BoundedRange.class)) {
                config.setQueryTree(timedExpandRanges(timers, "Expand Ranges", config.getQueryTree(), config, metadataHelper, scannerFactory));
            }
            // NOTE: GeoWavePruningVisitor should run before QueryPruningVisitor. If it runs after, there is a chance
            // that GeoWavePruningVisitor will prune all of the remaining indexed terms, which would leave a GeoWave
            // function without any indexed terms or ranges, which should evaluate to false. That case won't be handled
            // properly if we run GeoWavePruningVisitor after QueryPruningVisitor.
            config.setQueryTree(timedPruneGeoWaveTerms(timers, config.getQueryTree(), metadataHelper));
            if (reduceQuery) {
                config.setQueryTree(timedReduce(timers, "Reduce Query After Range Expansion", config.getQueryTree()));
            }
            // Check if there are functions that can be pushed into exceeded value ranges.
            if (nodeCount.hasAll(ASTFunctionNode.class, ExceededValueThresholdMarkerJexlNode.class)) {
                config.setQueryTree(timedPushFunctions(timers, config.getQueryTree(), config, metadataHelper));
            }
            if (executableExpansion) {
                config.setQueryTree(timedExecutableExpansion(timers, config.getQueryTree(), config, metadataHelper));
            }
            List<String> debugOutput = null;
            if (log.isDebugEnabled()) {
                debugOutput = new ArrayList<>(32);
            }
            // the terms to be delayed.
            if (!ExecutableDeterminationVisitor.isExecutable(config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields, debugOutput, metadataHelper)) {
                // if we now have an unexecutable tree because of delayed
                // predicates, then remove delayed predicates as needed and
                // reexpand
                config.setQueryTree(timedRemoveDelayedPredicates(timers, "Remove Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, indexLookupMap, scannerFactory, metadataHelper, debugOutput));
            }
            // if we now have an unexecutable tree because of missing
            // delayed predicates, then add delayed predicates where
            // possible
            config.setQueryTree(timedAddDelayedPredicates(timers, "Add Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, debugOutput));
        } catch (TableNotFoundException e) {
            stopwatch.stop();
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
            throw new DatawaveFatalQueryException(qe);
        } catch (CannotExpandUnfieldedTermFatalException e) {
            if (null != e.getCause() && e.getCause() instanceof DoNotPerformOptimizedQueryException)
                throw (DoNotPerformOptimizedQueryException) e.getCause();
            QueryException qe = new QueryException(DatawaveErrorCode.INDETERMINATE_INDEX_STATUS, e);
            throw new DatawaveFatalQueryException(qe);
        }
        stopwatch.stop();
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Bounded range and regex conversion has been disabled");
        }
    }
    return config.getQueryTree();
}

Example 2 with DoNotPerformOptimizedQueryException

use of datawave.query.exceptions.DoNotPerformOptimizedQueryException in project datawave by NationalSecurityAgency.

the class RegexIndexLookup method submit.

@Override
public synchronized void submit() {
    if (indexLookupMap == null) {
        indexLookupMap = new IndexLookupMap(config.getMaxUnfieldedExpansionThreshold(), config.getMaxValueExpansionThreshold());
        indexLookupMap.setPatterns(patterns);
        Multimap<String, Range> forwardMap = ArrayListMultimap.create(), reverseMap = ArrayListMultimap.create();
        // Loop over all the patterns, classifying them as forward or reverse index satisfiable
        Iterator<Entry<Key, Value>> iter = Iterators.emptyIterator();
        ScannerSession bs;
        IteratorSetting fairnessIterator = null;
        if (config.getMaxIndexScanTimeMillis() > 0) {
            // The fairness iterator solves the problem whereby we have runaway iterators as a result of an evaluation that never finds anything
            fairnessIterator = new IteratorSetting(1, TimeoutIterator.class);
            long maxTime = (long) (config.getMaxIndexScanTimeMillis() * 1.25);
            fairnessIterator.addOption(TimeoutIterator.MAX_SESSION_TIME, Long.valueOf(maxTime).toString());
        }
        for (String pattern : patterns) {
            if (config.getDisallowedRegexPatterns().contains(pattern)) {
                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.IGNORE_PATTERN_FOR_INDEX_LOOKUP, MessageFormat.format("Pattern: {0}", pattern));
                log.debug(qe);
                throw new DoNotPerformOptimizedQueryException(qe);
            }
            ShardIndexQueryTableStaticMethods.RefactoredRangeDescription rangeDescription;
            try {
                rangeDescription = ShardIndexQueryTableStaticMethods.getRegexRange(null, pattern, config.getFullTableScanEnabled(), helper, config);
            } catch (IllegalArgumentException | JavaRegexParseException e) {
                log.debug("Ignoring pattern that was not capable of being looked up in the index: " + pattern, e);
                continue;
            } catch (TableNotFoundException e) {
                log.error(e);
                throw new DatawaveFatalQueryException(e);
            } catch (ExecutionException e) {
                throw new DatawaveFatalQueryException(e);
            }
            if (log.isTraceEnabled()) {
                log.trace("Adding pattern " + pattern);
                log.trace("Adding pattern " + rangeDescription);
            }
            if (rangeDescription.isForReverseIndex) {
                reverseMap.put(pattern, rangeDescription.range);
            } else {
                forwardMap.put(pattern, rangeDescription.range);
            }
        }
        if (!fields.isEmpty() && !forwardMap.isEmpty()) {
            for (String key : forwardMap.keySet()) {
                Collection<Range> ranges = forwardMap.get(key);
                try {
                    bs = ShardIndexQueryTableStaticMethods.configureLimitedDiscovery(config, scannerFactory, config.getIndexTableName(), ranges, Collections.emptySet(), Collections.singleton(key), false, true);
                    bs.setResourceClass(BatchResource.class);
                } catch (Exception e) {
                    throw new DatawaveFatalQueryException(e);
                }
                SessionOptions opts = bs.getOptions();
                if (null != fairnessIterator) {
                    opts.addScanIterator(fairnessIterator);
                    IteratorSetting cfg = new IteratorSetting(config.getBaseIteratorPriority() + 100, TimeoutExceptionIterator.class);
                    opts.addScanIterator(cfg);
                }
                for (String field : fields) {
                    opts.fetchColumnFamily(new Text(field));
                }
                forwardLookupData.getSessions().add(bs);
                iter = Iterators.concat(iter, bs);
            }
            forwardLookupData.setTimedScanFuture(execService.submit(createTimedCallable(iter, fields, forwardLookupData, indexLookupMap)));
        }
        if (!reverseFields.isEmpty() && !reverseMap.isEmpty()) {
            for (String key : reverseMap.keySet()) {
                Collection<Range> ranges = reverseMap.get(key);
                if (log.isTraceEnabled()) {
                    log.trace("adding " + ranges + " for reverse");
                }
                try {
                    bs = ShardIndexQueryTableStaticMethods.configureLimitedDiscovery(config, scannerFactory, config.getReverseIndexTableName(), ranges, Collections.emptySet(), Collections.singleton(key), true, true);
                    bs.setResourceClass(BatchResource.class);
                } catch (Exception e) {
                    throw new DatawaveFatalQueryException(e);
                }
                SessionOptions opts = bs.getOptions();
                if (null != fairnessIterator) {
                    opts.addScanIterator(fairnessIterator);
                    opts.addScanIterator(new IteratorSetting(config.getBaseIteratorPriority() + 100, TimeoutExceptionIterator.class));
                }
                for (String field : reverseFields) {
                    opts.fetchColumnFamily(new Text(field));
                }
                reverseLookupData.getSessions().add(bs);
                iter = Iterators.concat(iter, bs);
            }
            reverseLookupData.setTimedScanFuture(execService.submit(createTimedCallable(iter, reverseFields, reverseLookupData, indexLookupMap)));
        }
    }
}

Example 3 with DoNotPerformOptimizedQueryException

use of datawave.query.exceptions.DoNotPerformOptimizedQueryException in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTableStaticMethods method getRegexRange.

/**
 * Get a range description for a specified query term, which could be a regex. Note that it is assumed that the column family set will include the
 * fieldname.
 *
 * @param fieldName
 * @param normalizedQueryTerm
 * @param fullTableScanEnabled
 * @param metadataHelper
 * @param config
 * @return
 * @throws datawave.query.parser.JavaRegexAnalyzer.JavaRegexParseException
 * @throws org.apache.accumulo.core.client.TableNotFoundException
 * @throws java.util.concurrent.ExecutionException
 */
public static RefactoredRangeDescription getRegexRange(String fieldName, String normalizedQueryTerm, boolean fullTableScanEnabled, MetadataHelper metadataHelper, ShardQueryConfiguration config) throws JavaRegexAnalyzer.JavaRegexParseException, TableNotFoundException, ExecutionException {
    if (log.isDebugEnabled()) {
        log.debug("getRegexRange: " + normalizedQueryTerm);
    }
    RefactoredRangeDescription rangeDesc = new RefactoredRangeDescription();
    JavaRegexAnalyzer regex = new JavaRegexAnalyzer(normalizedQueryTerm);
    // If we have a leading wildcard, reverse the term and use the global reverse index.
    if (shouldUseReverseIndex(regex, fieldName, metadataHelper, config)) {
        rangeDesc.isForReverseIndex = true;
        if (!regex.isTrailingLiteral()) {
            // if we require a full table scan but it is disabled, then bail
            if (!fullTableScanEnabled) {
                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.WILDCARDS_BOTH_SIDES, MessageFormat.format("Term: {0}", normalizedQueryTerm));
                log.error(qe);
                throw new DoNotPerformOptimizedQueryException(qe);
            }
            rangeDesc.range = new Range();
        } else {
            StringBuilder buf = new StringBuilder(regex.getTrailingLiteral());
            String reversedQueryTerm = buf.reverse().toString();
            if (log.isTraceEnabled()) {
                StringBuilder sb = new StringBuilder(256);
                sb.append("For '").append(normalizedQueryTerm).append("'");
                if (null != fieldName) {
                    sb.append(" in the field ").append(fieldName);
                }
                sb.append(", using the reverse index with the literal: '").append(reversedQueryTerm).append("'");
                log.trace(sb.toString());
            }
            Key startKey = new Key(reversedQueryTerm);
            Key endKey = new Key(reversedQueryTerm + Constants.MAX_UNICODE_STRING);
            // set the upper and lower bounds
            rangeDesc.range = new Range(startKey, false, endKey, false);
        }
    } else {
        rangeDesc.isForReverseIndex = false;
        if (!regex.isLeadingLiteral()) {
            // if we require a full table scan but it is disabled, then bail
            if (!fullTableScanEnabled) {
                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.WILDCARDS_BOTH_SIDES, MessageFormat.format("Term: {0}", normalizedQueryTerm));
                log.error(qe);
                throw new DoNotPerformOptimizedQueryException(qe);
            }
            rangeDesc.range = new Range();
        } else {
            String queryTerm = regex.getLeadingLiteral();
            if (log.isTraceEnabled()) {
                StringBuilder sb = new StringBuilder(256);
                sb.append("For '").append(normalizedQueryTerm).append("'");
                if (null != fieldName) {
                    sb.append(" in the field ").append(fieldName);
                }
                sb.append(", using the forward index with the literal: '").append(queryTerm).append("'");
                log.trace(sb.toString());
            }
            Key startKey = new Key(queryTerm);
            Key endKey = new Key(queryTerm + Constants.MAX_UNICODE_STRING);
            // either middle or trailing wildcard, truncate the field value at the wildcard location
            // for upper bound, tack on the upper bound UTF character
            rangeDesc.range = new Range(startKey, false, endKey, false);
        }
    }
    return rangeDesc;
}