Examples with DatawaveFatalQueryException datawave.query.exceptions.DatawaveFatalQueryException used on opensource projects

Example 36 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class EdgeQueryLogic method normalizeJexlQuery.

/**
 * Method to expand the values supplied for SOURCE or SINK into all of the permutations after normalizing
 *
 * @param query
 * @return
 */
protected VisitationContext normalizeJexlQuery(String query, boolean getFullNormalizedQuery) {
    if (visitationContext == null) {
        throw new DatawaveFatalQueryException("Something went wrong running your query");
    }
    // normalizing the query we want to fail here instead of over on the server side
    if (!visitationContext.getNormalizedQuery().toString().equals("")) {
        try {
            JexlASTHelper.parseJexlQuery(visitationContext.getNormalizedQuery().toString());
        } catch (ParseException e) {
            log.error("Could not parse JEXL AST after performing transformations to run the query. Normalized Stats Query: " + visitationContext.getNormalizedStatsQuery(), e);
            throw new DatawaveFatalQueryException("Something went wrong running your query", e);
        }
    }
    if (!visitationContext.getNormalizedStatsQuery().toString().equals("")) {
        try {
            JexlASTHelper.parseJexlQuery(visitationContext.getNormalizedStatsQuery().toString());
        } catch (ParseException e) {
            log.error("Could not parse JEXL AST after performing transformations to run the query. Normalized Stats Query: " + visitationContext.getNormalizedStatsQuery(), e);
            throw new DatawaveFatalQueryException("Something went wrong running your query", e);
        }
    }
    pruneAndSetPreFilterValues(visitationContext.getPreFilterValues());
    long termCount = visitationContext.getTermCount();
    if (termCount > config.getMaxQueryTerms()) {
        throw new IllegalArgumentException("Edge query max terms limit (" + config.getMaxQueryTerms() + ") exceeded: " + termCount + ".");
    }
    return visitationContext;
}

Example 37 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class JexlNodeFactory method buildUntypedNewNode.

/**
 * Build a JexlNode with a Number literal
 *
 * @param fieldName
 * @param fieldValue
 * @return
 */
protected static JexlNode buildUntypedNewNode(JexlNode newNode, ASTIdentifier fieldName, Number fieldValue) {
    ASTNumberLiteral literal = new ASTNumberLiteral(ParserTreeConstants.JJTNUMBERLITERAL);
    literal.image = fieldValue.toString();
    if (NATURAL_NUMBERS.contains(fieldValue.getClass())) {
        literal.setNatural(fieldValue.toString());
    } else if (REAL_NUMBERS.contains(fieldValue.getClass())) {
        literal.setReal(fieldValue.toString());
    } else {
        QueryException qe = new QueryException(DatawaveErrorCode.ASTNUMBERLITERAL_TYPE_ASCERTAIN_ERROR, MessageFormat.format("{0}", literal));
        throw new DatawaveFatalQueryException(qe);
    }
    return buildUntypedNewNode(newNode, fieldName, literal);
}

Example 38 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class BooleanChunkingQueryPlanner method process.

@Override
protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    final ArrayList<QueryData> data = Lists.newArrayList();
    ASTJexlScript queryTree = updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings);
    if (queryTree == null) {
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    String newQueryString = JexlStringBuildingVisitor.buildQuery(queryTree);
    if (StringUtils.isBlank(newQueryString)) {
        throw new DatawaveFatalQueryException("Query string after query modification was empty. Cannot run query.");
    } else if (log.isDebugEnabled()) {
        log.debug("Final query passed to QueryIterator:");
        log.debug(newQueryString);
    }
    queryData.setQuery(newQueryString);
    data.add(queryData);
    return new CloseableIterable<QueryData>() {

        @Override
        public Iterator<QueryData> iterator() {
            return data.iterator();
        }

        @Override
        public void close() throws IOException {
        }
    };
}

Example 39 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method processTree.

protected ASTJexlScript processTree(final ASTJexlScript originalQueryTree, ShardQueryConfiguration config, Query settings, MetadataHelper metadataHelper, ScannerFactory scannerFactory, QueryData queryData, QueryStopwatch timers, QueryModel queryModel) throws DatawaveQueryException {
    config.setQueryTree(originalQueryTree);
    TraceStopwatch stopwatch = null;
    if (!disableWhindexFieldMappings) {
        // apply the value-specific field mappings for GeoWave functions
        config.setQueryTree(timedApplyWhindexFieldMappings(timers, config.getQueryTree(), config, metadataHelper, settings));
    }
    if (!disableExpandIndexFunction) {
        // expand the index queries for the functions
        config.setQueryTree(timedExpandIndexQueriesForFunctions(timers, config.getQueryTree(), config, metadataHelper));
    }
    // apply the node transform rules
    // running it here before any unfielded expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Unfielded Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // left as a regex
    if (!disableAnyFieldLookup) {
        config.setQueryTree(timedExpandAnyFieldRegexNodes(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, settings.getQuery()));
    }
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query After ANYFIELD Expansions", config.getQueryTree()));
    }
    if (!disableTestNonExistentFields) {
        timedTestForNonExistentFields(timers, config.getQueryTree(), config, metadataHelper, queryModel, settings);
    }
    // apply the node transform rules
    // running it here before any regex or range expansions to enable potentially pushing down terms before index lookups
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Regex/Range Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    timedFetchDatatypes(timers, "Fetch Required Datatypes", config.getQueryTree(), config);
    config.setQueryTree(timedFixUnindexedNumerics(timers, config.getQueryTree(), config));
    config.setQueryTree(timedExpandMultiNormalizedTerms(timers, config.getQueryTree(), config, metadataHelper));
    // if we have any index holes, then mark em
    if (!config.getIndexHoles().isEmpty()) {
        config.setQueryTree(timedMarkIndexHoles(timers, config.getQueryTree(), config, metadataHelper));
    }
    // lets precompute the indexed fields and index only fields for the specific datatype if needed below
    Set<String> indexedFields = null;
    Set<String> indexOnlyFields = null;
    Set<String> nonEventFields = null;
    if (config.getMinSelectivity() > 0 || !disableBoundedLookup) {
        try {
            indexedFields = metadataHelper.getIndexedFields(config.getDatatypeFilter());
            indexOnlyFields = metadataHelper.getIndexOnlyFields(config.getDatatypeFilter());
            nonEventFields = metadataHelper.getNonEventFields(config.getDatatypeFilter());
        } catch (TableNotFoundException te) {
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, te);
            throw new DatawaveFatalQueryException(qe);
        }
    }
    // apply the node transform rules
    config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Pushdown/Pullup Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
    // push down terms that are over the min selectivity
    if (config.getMinSelectivity() > 0) {
        config.setQueryTree(timedPushdownLowSelectiveTerms(timers, config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields));
    }
    config.setQueryTree(timedForceFieldToFieldComparison(timers, config.getQueryTree()));
    if (!disableCompositeFields) {
        config.setQueryTree(timedExpandCompositeFields(timers, config.getQueryTree(), config));
    }
    if (!disableBoundedLookup) {
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Expand bounded query ranges (total)");
        // Expand any bounded ranges into a conjunction of discrete terms
        try {
            Map<String, IndexLookup> indexLookupMap = new HashMap<>();
            // Check if there is any regex to expand.
            NodeTypeCount nodeCount = NodeTypeCountVisitor.countNodes(config.getQueryTree());
            if (nodeCount.hasAny(ASTNRNode.class, ASTERNode.class)) {
                config.setQueryTree(timedExpandRegex(timers, "Expand Regex", config.getQueryTree(), config, metadataHelper, scannerFactory, indexLookupMap));
            }
            // Check if there are any bounded ranges to expand.
            if (nodeCount.isPresent(BoundedRange.class)) {
                config.setQueryTree(timedExpandRanges(timers, "Expand Ranges", config.getQueryTree(), config, metadataHelper, scannerFactory));
            }
            // NOTE: GeoWavePruningVisitor should run before QueryPruningVisitor. If it runs after, there is a chance
            // that GeoWavePruningVisitor will prune all of the remaining indexed terms, which would leave a GeoWave
            // function without any indexed terms or ranges, which should evaluate to false. That case won't be handled
            // properly if we run GeoWavePruningVisitor after QueryPruningVisitor.
            config.setQueryTree(timedPruneGeoWaveTerms(timers, config.getQueryTree(), metadataHelper));
            if (reduceQuery) {
                config.setQueryTree(timedReduce(timers, "Reduce Query After Range Expansion", config.getQueryTree()));
            }
            // Check if there are functions that can be pushed into exceeded value ranges.
            if (nodeCount.hasAll(ASTFunctionNode.class, ExceededValueThresholdMarkerJexlNode.class)) {
                config.setQueryTree(timedPushFunctions(timers, config.getQueryTree(), config, metadataHelper));
            }
            if (executableExpansion) {
                config.setQueryTree(timedExecutableExpansion(timers, config.getQueryTree(), config, metadataHelper));
            }
            List<String> debugOutput = null;
            if (log.isDebugEnabled()) {
                debugOutput = new ArrayList<>(32);
            }
            // the terms to be delayed.
            if (!ExecutableDeterminationVisitor.isExecutable(config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields, debugOutput, metadataHelper)) {
                // if we now have an unexecutable tree because of delayed
                // predicates, then remove delayed predicates as needed and
                // reexpand
                config.setQueryTree(timedRemoveDelayedPredicates(timers, "Remove Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, indexLookupMap, scannerFactory, metadataHelper, debugOutput));
            }
            // if we now have an unexecutable tree because of missing
            // delayed predicates, then add delayed predicates where
            // possible
            config.setQueryTree(timedAddDelayedPredicates(timers, "Add Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, debugOutput));
        } catch (TableNotFoundException e) {
            stopwatch.stop();
            QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
            throw new DatawaveFatalQueryException(qe);
        } catch (CannotExpandUnfieldedTermFatalException e) {
            if (null != e.getCause() && e.getCause() instanceof DoNotPerformOptimizedQueryException)
                throw (DoNotPerformOptimizedQueryException) e.getCause();
            QueryException qe = new QueryException(DatawaveErrorCode.INDETERMINATE_INDEX_STATUS, e);
            throw new DatawaveFatalQueryException(qe);
        }
        stopwatch.stop();
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Bounded range and regex conversion has been disabled");
        }
    }
    return config.getQueryTree();
}

Example 40 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedExpandCompositeFields.

protected ASTJexlScript timedExpandCompositeFields(QueryStopwatch timers, ASTJexlScript script, ShardQueryConfiguration config) throws DatawaveQueryException {
    try {
        config.setCompositeToFieldMap(metadataHelper.getCompositeToFieldMap(config.getDatatypeFilter()));
        config.setCompositeTransitionDates(metadataHelper.getCompositeTransitionDateMap(config.getDatatypeFilter()));
        config.setCompositeFieldSeparators(metadataHelper.getCompositeFieldSeparatorMap(config.getDatatypeFilter()));
        config.setFieldToDiscreteIndexTypes(CompositeUtils.getFieldToDiscreteIndexTypeMap(config.getQueryFieldsDatatypes()));
    } catch (TableNotFoundException e) {
        QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
        throw new DatawaveFatalQueryException(qe);
    }
    return visitorManager.timedVisit(timers, "Expand Composite Terms", () -> (ExpandCompositeTerms.expandTerms(config, script)));
}