Examples with DatawaveQueryException datawave.query.exceptions.DatawaveQueryException used on opensource projects

Example 1 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedExpandRanges.

private ASTJexlScript timedExpandRanges(QueryStopwatch timers, String stage, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, ScannerFactory scannerFactory) throws DatawaveQueryException {
    config.setQueryTree(script);
    TraceStopwatch innerStopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    try {
        config.setQueryTree(BoundedRangeIndexExpansionVisitor.expandBoundedRanges(config, scannerFactory, metadataHelper, config.getQueryTree()));
    } catch (TableNotFoundException e) {
        throw new DatawaveQueryException("Failed to Expand Ranges", e);
    }
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Query after expanding ranges:");
    }
    innerStopwatch.stop();
    return config.getQueryTree();
}

Example 2 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 3 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

@Override
public CloseableIterable<QueryData> process(GenericQueryConfiguration genericConfig, String query, Query settings, ScannerFactory scannerFactory) throws DatawaveQueryException {
    if (!(genericConfig instanceof ShardQueryConfiguration)) {
        throw new ClassCastException("Config object must be an instance of ShardQueryConfiguration");
    }
    builderThread = Executors.newSingleThreadExecutor();
    ShardQueryConfiguration config = (ShardQueryConfiguration) genericConfig;
    // lets mark the query as started (used by ivarators at a minimum)
    try {
        markQueryStarted(config, settings);
    } catch (Exception e) {
        throw new DatawaveQueryException("Failed to mark query as started" + settings.getId(), e);
    }
    return process(scannerFactory, getMetadataHelper(config), getDateIndexHelper(config), config, query, settings);
}

Example 4 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedTestForNonExistentFields.

protected void timedTestForNonExistentFields(QueryStopwatch timers, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, QueryModel queryModel, Query settings) throws DatawaveQueryException {
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Test for Non-Existent Fields");
    // Verify that the query does not contain fields we've never seen
    // before
    Set<String> specialFields = Sets.newHashSet(QueryOptions.DEFAULT_DATATYPE_FIELDNAME, Constants.ANY_FIELD, Constants.NO_FIELD);
    specialFields.addAll(config.getEvaluationOnlyFields());
    Set<String> nonexistentFields = FieldMissingFromSchemaVisitor.getNonExistentFields(metadataHelper, script, config.getDatatypeFilter(), specialFields);
    if (log.isDebugEnabled()) {
        log.debug("Testing for non-existent fields, found: " + nonexistentFields.size());
    }
    // ensure that all of the fields actually exist in the data dictionary
    Set<String> allFields = null;
    try {
        allFields = metadataHelper.getAllFields(config.getDatatypeFilter());
    } catch (TableNotFoundException e) {
        throw new DatawaveQueryException("Unable get get data dictionary", e);
    }
    // Fields in the data dictionary is always uppercase. Convert the unique fields to uppercase
    // so the comparisons are case insensitive
    List<String> fields = config.getUniqueFields().getFields().stream().map(String::toUpperCase).collect(Collectors.toList());
    // for the unique fields we need to also look for any model aliases (forward or reverse) and fields generated post evaluation (e.g. HIT_TERM)
    // this is because unique fields operate on the fields as returned to the user. We essentially leave all variants of the fields
    // in the unique field list to ensure we catch everything
    Set<String> uniqueFields = new HashSet<>(allFields);
    if (queryModel != null) {
        uniqueFields.addAll(queryModel.getForwardQueryMapping().keySet());
        uniqueFields.addAll(queryModel.getReverseQueryMapping().values());
    }
    uniqueFields.add(JexlEvaluation.HIT_TERM_FIELD);
    if (!uniqueFields.containsAll(fields)) {
        Set<String> missingFields = Sets.newHashSet(config.getUniqueFields().getFields());
        missingFields.removeAll(uniqueFields);
        nonexistentFields.addAll(missingFields);
    }
    if (!nonexistentFields.isEmpty()) {
        String datatypeFilterSet = (null == config.getDatatypeFilter()) ? "none" : config.getDatatypeFilter().toString();
        if (log.isTraceEnabled()) {
            try {
                log.trace("current size of fields" + metadataHelper.getAllFields(config.getDatatypeFilter()));
                log.trace("all fields: " + metadataHelper.getAllFields(config.getDatatypeFilter()));
            } catch (TableNotFoundException e) {
                log.error("table not found when reading metadata", e);
            }
            log.trace("QueryModel:" + (null == queryModel ? "null" : queryModel));
            log.trace("metadataHelper " + metadataHelper);
        }
        log.trace("QueryModel:" + (null == queryModel ? "null" : queryModel));
        log.trace("metadataHelper " + metadataHelper);
        BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.FIELDS_NOT_IN_DATA_DICTIONARY, MessageFormat.format("Datatype Filter: {0}, Missing Fields: {1}, Auths: {2}", datatypeFilterSet, nonexistentFields, settings.getQueryAuthorizations()));
        log.error(qe);
        throw new InvalidQueryException(qe);
    }
    stopwatch.stop();
}

Example 5 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedExpandAnyFieldRegexNodes.

protected ASTJexlScript timedExpandAnyFieldRegexNodes(QueryStopwatch timers, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, ScannerFactory scannerFactory, String query) throws DatawaveQueryException {
    try {
        config.setIndexedFields(metadataHelper.getIndexedFields(config.getDatatypeFilter()));
        config.setReverseIndexedFields(metadataHelper.getReverseIndexedFields(config.getDatatypeFilter()));
        // @formatter:off
        return visitorManager.timedVisit(timers, "Expand ANYFIELD Regex Nodes", () -> {
            try {
                return UnfieldedIndexExpansionVisitor.expandUnfielded(config, scannerFactory, metadataHelper, script);
            } catch (InstantiationException | IllegalAccessException | TableNotFoundException e) {
                // rethrow as a datawave query exception because method contracts
                throw new DatawaveQueryException(e);
            }
        });
    // @formatter:on
    } catch (EmptyUnfieldedTermExpansionException e) {
        // The visitor will only throw this if we cannot expand anything resulting in empty query
        NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.UNFIELDED_QUERY_ZERO_MATCHES, e, MessageFormat.format("Query: ", query));
        log.info(qe);
        throw new NoResultsException(qe);
    } catch (TableNotFoundException e) {
        QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
        log.info(qe);
        throw new DatawaveFatalQueryException(qe);
    }
}