Examples with QueryException datawave.webservice.query.exception.QueryException used on opensource projects

Example 26 with QueryException

use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.

the class ModelBean method deleteMapping.

private VoidResponse deleteMapping(datawave.webservice.model.Model model, String modelTableName, boolean reloadCache) {
    VoidResponse response = new VoidResponse();
    Connector connector = null;
    BatchWriter writer = null;
    String tableName = this.checkModelTableName(modelTableName);
    try {
        Map<String, String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
        connector = connectionFactory.getConnection(AccumuloConnectionFactory.Priority.LOW, trackingMap);
        writer = connector.createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.MILLISECONDS).setMaxMemory(BATCH_WRITER_MAX_MEMORY).setMaxWriteThreads(BATCH_WRITER_MAX_THREADS));
        for (FieldMapping mapping : model.getFields()) {
            Mutation m = ModelKeyParser.createDeleteMutation(mapping, model.getName());
            writer.addMutation(m);
        }
    } catch (Exception e) {
        log.error("Could not delete mapping.", e);
        QueryException qe = new QueryException(DatawaveErrorCode.MAPPING_DELETION_ERROR, e);
        response.addException(qe.getBottomQueryException());
        throw new DatawaveWebApplicationException(qe, response);
    } finally {
        if (null != writer) {
            try {
                writer.close();
            } catch (MutationsRejectedException e1) {
                QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1);
                log.error(qe);
                response.addException(qe);
                throw new DatawaveWebApplicationException(qe, response);
            }
        }
        if (null != connector) {
            try {
                connectionFactory.returnConnection(connector);
            } catch (Exception e) {
                log.error("Error returning connection to factory", e);
            }
        }
    }
    if (reloadCache)
        cache.reloadCache(tableName);
    return response;
}

Example 27 with QueryException

use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method process.

protected CloseableIterable<QueryData> process(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, Query settings) throws DatawaveQueryException {
    final QueryData queryData = new QueryData();
    settingFuture = null;
    IteratorSetting cfg = null;
    if (preloadOptions) {
        cfg = getQueryIterator(metadataHelper, config, settings, "", false);
    }
    try {
        config.setQueryTree(updateQueryTree(scannerFactory, metadataHelper, dateIndexHelper, config, query, queryData, settings));
    } catch (StackOverflowError e) {
        if (log.isTraceEnabled()) {
            log.trace("Stack trace for overflow " + e);
        }
        PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, e);
        log.warn(qe);
        throw new DatawaveFatalQueryException(qe);
    } catch (NoResultsException e) {
        if (log.isTraceEnabled()) {
            log.trace("Definitively determined that no results exist from the indexes");
        }
        return DefaultQueryPlanner.emptyCloseableIterator();
    }
    boolean isFullTable = false;
    Tuple2<CloseableIterable<QueryPlan>, Boolean> queryRanges = null;
    if (!config.isGeneratePlanOnly()) {
        queryRanges = getQueryRanges(scannerFactory, metadataHelper, config, config.getQueryTree());
        // a full table scan is required if
        isFullTable = queryRanges.second();
        // abort if we cannot handle full table scans
        if (isFullTable && !config.getFullTableScanEnabled()) {
            PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
    }
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Rebuild JEXL String from AST");
    // Set the final query after we're done mucking with it
    String newQueryString = JexlStringBuildingVisitor.buildQuery(config.getQueryTree());
    if (log.isTraceEnabled())
        log.trace("newQueryString is " + newQueryString);
    if (StringUtils.isBlank(newQueryString)) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.EMPTY_QUERY_STRING_AFTER_MODIFICATION);
        throw new DatawaveFatalQueryException(qe);
    }
    stopwatch.stop();
    stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Construct IteratorSettings");
    queryData.setQuery(newQueryString);
    if (!config.isGeneratePlanOnly()) {
        while (null == cfg) {
            cfg = getQueryIterator(metadataHelper, config, settings, "", false);
        }
        configureIterator(config, cfg, newQueryString, isFullTable);
    }
    // Load the IteratorSettings into the QueryData instance
    queryData.setSettings(Lists.newArrayList(cfg));
    stopwatch.stop();
    this.plannedScript = newQueryString;
    config.setQueryString(this.plannedScript);
    // docsToCombineForEvaluation is only enabled when threading is used
    if (config.getMaxEvaluationPipelines() == 1)
        docsToCombineForEvaluation = -1;
    if (!config.isGeneratePlanOnly()) {
        // add the geo query comparator to sort by geo range granularity if this is a geo query
        List<Comparator<QueryPlan>> queryPlanComparators = null;
        if (config.isSortGeoWaveQueryRanges()) {
            List<String> geoFields = new ArrayList<>();
            for (String fieldName : config.getIndexedFields()) {
                for (Type type : config.getQueryFieldsDatatypes().get(fieldName)) {
                    if (type instanceof AbstractGeometryType) {
                        geoFields.add(fieldName);
                        break;
                    }
                }
            }
            if (!geoFields.isEmpty()) {
                queryPlanComparators = new ArrayList<>();
                queryPlanComparators.add(new GeoWaveQueryPlanComparator(geoFields));
                queryPlanComparators.add(new DefaultQueryPlanComparator());
            }
        }
        // @formatter:off
        return new ThreadedRangeBundler.Builder().setOriginal(queryData).setQueryTree(config.getQueryTree()).setRanges(queryRanges.first()).setMaxRanges(maxRangesPerQueryPiece()).setDocsToCombine(docsToCombineForEvaluation).setSettings(settings).setDocSpecificLimitOverride(docSpecificOverride).setMaxRangeWaitMillis(maxRangeWaitMillis).setQueryPlanComparators(queryPlanComparators).setNumRangesToBuffer(config.getNumRangesToBuffer()).setRangeBufferTimeoutMillis(config.getRangeBufferTimeoutMillis()).setRangeBufferPollMillis(config.getRangeBufferPollMillis()).build();
    // @formatter:on
    } else {
        return null;
    }
}

Example 28 with QueryException

use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedExpandAnyFieldRegexNodes.

protected ASTJexlScript timedExpandAnyFieldRegexNodes(QueryStopwatch timers, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, ScannerFactory scannerFactory, String query) throws DatawaveQueryException {
    try {
        config.setIndexedFields(metadataHelper.getIndexedFields(config.getDatatypeFilter()));
        config.setReverseIndexedFields(metadataHelper.getReverseIndexedFields(config.getDatatypeFilter()));
        // @formatter:off
        return visitorManager.timedVisit(timers, "Expand ANYFIELD Regex Nodes", () -> {
            try {
                return UnfieldedIndexExpansionVisitor.expandUnfielded(config, scannerFactory, metadataHelper, script);
            } catch (InstantiationException | IllegalAccessException | TableNotFoundException e) {
                // rethrow as a datawave query exception because method contracts
                throw new DatawaveQueryException(e);
            }
        });
    // @formatter:on
    } catch (EmptyUnfieldedTermExpansionException e) {
        // The visitor will only throw this if we cannot expand anything resulting in empty query
        NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.UNFIELDED_QUERY_ZERO_MATCHES, e, MessageFormat.format("Query: ", query));
        log.info(qe);
        throw new NoResultsException(qe);
    } catch (TableNotFoundException e) {
        QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
        log.info(qe);
        throw new DatawaveFatalQueryException(qe);
    }
}

Example 29 with QueryException

use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedCheckForTokenizedFields.

protected void timedCheckForTokenizedFields(QueryStopwatch timers, String stage, final ASTJexlScript script, ShardQueryConfiguration config, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper) {
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - " + stage);
    // Figure out if the query contained any term frequency terms so we know
    // if we may use the term frequencies instead of the fields index in some cases
    Set<String> queryTfFields = Collections.emptySet();
    Set<String> termFrequencyFields;
    try {
        termFrequencyFields = metadataHelper.getTermFrequencyFields(config.getDatatypeFilter());
    } catch (TableNotFoundException e) {
        stopwatch.stop();
        QueryException qe = new QueryException(DatawaveErrorCode.TERM_FREQUENCY_FIELDS_RETRIEVAL_ERROR, e);
        throw new DatawaveFatalQueryException(qe);
    }
    if (!termFrequencyFields.isEmpty()) {
        queryTfFields = SetMembershipVisitor.getMembers(termFrequencyFields, config, metadataHelper, dateIndexHelper, config.getQueryTree());
        // Print the nice log message
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Computed that the query " + (queryTfFields.isEmpty() ? " does not contain any " : "contains " + queryTfFields + " as ") + " term frequency field(s)");
        }
    }
    config.setQueryTermFrequencyFields(queryTfFields);
    // now determine if we actually require gathering term frequencies
    if (!queryTfFields.isEmpty()) {
        Multimap<String, Function> contentFunctions = TermOffsetPopulator.getContentFunctions(config.getQueryTree());
        config.setTermFrequenciesRequired(!contentFunctions.isEmpty());
        // Print the nice log message
        if (log.isDebugEnabled()) {
            logQuery(config.getQueryTree(), "Computed that the query " + (contentFunctions.isEmpty() ? " does not require " : "requires") + " term frequency lookup");
        }
    }
    stopwatch.stop();
}

Example 30 with QueryException

use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method getQueryRanges.

/**
 * Returns a Tuple2<Iterable<Range>,Boolean> whose elements represent the Ranges to use for querying the shard table and whether or not this is
 * a "full-table-scan" query.
 *
 * @param scannerFactory
 * @param metadataHelper
 * @param config
 * @param queryTree
 * @return
 * @throws DatawaveQueryException
 */
public Tuple2<CloseableIterable<QueryPlan>, Boolean> getQueryRanges(ScannerFactory scannerFactory, MetadataHelper metadataHelper, ShardQueryConfiguration config, JexlNode queryTree) throws DatawaveQueryException {
    Preconditions.checkNotNull(queryTree);
    boolean needsFullTable = false;
    CloseableIterable<QueryPlan> ranges = null;
    // if the query has already been reduced to false there is no reason to do more
    if (QueryPruningVisitor.getState(queryTree) == QueryPruningVisitor.TruthState.FALSE) {
        return new Tuple2<>(emptyCloseableIterator(), false);
    }
    // if we still have an unexecutable tree, then a full table scan is
    // required
    List<String> debugOutput = null;
    if (log.isDebugEnabled()) {
        debugOutput = new ArrayList<>(32);
    }
    STATE state = ExecutableDeterminationVisitor.getState(queryTree, config, metadataHelper, debugOutput);
    if (log.isDebugEnabled()) {
        logDebug(debugOutput, "ExecutableDeterminationVisitor at getQueryRanges:");
    }
    if (state != STATE.EXECUTABLE) {
        if (state == STATE.ERROR) {
            log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
            BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
            throw new InvalidQueryException(qe);
        }
        log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
        needsFullTable = true;
    }
    // scan, then lets try to compute ranges
    if (!needsFullTable) {
        // count the terms
        int termCount = TermCountingVisitor.countTerms(queryTree);
        if (termCount >= pushdownThreshold) {
            if (log.isTraceEnabled()) {
                log.trace("pushing down query because it has " + termCount + " when our max is " + pushdownThreshold);
            }
            config.setCollapseUids(true);
        }
        TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("DefaultQueryPlanner - Begin stream of ranges from inverted index");
        RangeStream stream = initializeRangeStream(config, scannerFactory, metadataHelper);
        ranges = stream.streamPlans(queryTree);
        if (log.isTraceEnabled()) {
            log.trace("query stream is " + stream.context());
        }
        // if a term threshold is exceeded and we cannot handle that, then
        // throw unsupported
        boolean thresholdExceeded = StreamContext.EXCEEDED_TERM_THRESHOLD.equals(stream.context());
        if (thresholdExceeded && !config.canHandleExceededTermThreshold()) {
            throw new UnsupportedOperationException(EXCEED_TERM_EXPANSION_ERROR);
        }
        if (StreamContext.UNINDEXED.equals(stream.context())) {
            log.debug("Needs full table scan because of unindexed fields");
            needsFullTable = true;
        } else if (StreamContext.DELAYED_FIELD.equals(stream.context())) {
            log.debug("Needs full table scan because query consists of only delayed expressions");
            needsFullTable = true;
        } else // force a full table scan
        if (IvaratorRequiredVisitor.isIvaratorRequired(queryTree) && !config.canHandleExceededValueThreshold()) {
            log.debug("Needs full table scan because we exceeded the value threshold and config.canHandleExceededValueThreshold() is false");
            needsFullTable = true;
        }
        stopwatch.stop();
    }
    if (needsFullTable) {
        if (config.getFullTableScanEnabled()) {
            ranges = this.getFullScanRange(config, queryTree);
        } else {
            if (log.isTraceEnabled())
                log.trace("Full table scans are not enabled, query will not be run");
            QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
            throw new FullTableScansDisallowedException(qe);
        }
        if (log.isTraceEnabled())
            log.trace("Ranges are " + ranges);
    }
    return new Tuple2<>(ranges, needsFullTable);
}