Examples with DatawaveQueryException datawave.query.exceptions.DatawaveQueryException used on opensource projects

Example 11 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedExpandCompositeFields.

protected ASTJexlScript timedExpandCompositeFields(QueryStopwatch timers, ASTJexlScript script, ShardQueryConfiguration config) throws DatawaveQueryException {
    try {
        config.setCompositeToFieldMap(metadataHelper.getCompositeToFieldMap(config.getDatatypeFilter()));
        config.setCompositeTransitionDates(metadataHelper.getCompositeTransitionDateMap(config.getDatatypeFilter()));
        config.setCompositeFieldSeparators(metadataHelper.getCompositeFieldSeparatorMap(config.getDatatypeFilter()));
        config.setFieldToDiscreteIndexTypes(CompositeUtils.getFieldToDiscreteIndexTypeMap(config.getQueryFieldsDatatypes()));
    } catch (TableNotFoundException e) {
        QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
        throw new DatawaveFatalQueryException(qe);
    }
    return visitorManager.timedVisit(timers, "Expand Composite Terms", () -> (ExpandCompositeTerms.expandTerms(config, script)));
}

Example 12 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method addDateFilters.

/**
 * Adding date filters if the query parameters specify that the dates are to be other than the default
 *
 * @param queryTree
 * @param scannerFactory
 * @param metadataHelper
 * @param config
 * @return the updated query tree
 * @throws TableNotFoundException
 */
public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, Query settings) throws TableNotFoundException, DatawaveQueryException {
    String defaultDateType = config.getDefaultDateTypeName();
    String dateType = defaultDateType;
    Parameter dateTypeParameter = settings.findParameter(QueryParameters.DATE_RANGE_TYPE);
    if (dateTypeParameter != null && dateTypeParameter.getParameterValue() != null) {
        String parm = dateTypeParameter.getParameterValue().trim();
        if (!parm.isEmpty()) {
            dateType = parm.toUpperCase();
        }
    }
    // time, then we need to modify the query
    if (!dateType.equals(defaultDateType)) {
        log.info("Using the date index for " + dateType);
        // if no date index helper configured, then we are in error
        if (dateIndexHelper == null) {
            throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
        }
        // get all of the fields used for this date type
        DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(), config.getDatatypeFilter());
        if (dateIndexData.getFields().isEmpty()) {
            log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
            // If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
            // Also it will be understandable from the plan as to why no results were returned.
            dateIndexData.getFields().add(Constants.NO_FIELD);
        }
        log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
        // now for each field, add an expression to filter that date
        List<JexlNode> andChildren = new ArrayList<>();
        for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
            andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
        }
        List<JexlNode> orChildren = new ArrayList<>();
        for (String field : dateIndexData.getFields()) {
            orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
        }
        if (orChildren.size() > 1) {
            andChildren.add(JexlNodeFactory.createOrNode(orChildren));
        } else {
            andChildren.addAll(orChildren);
        }
        JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
        JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));
        // now lets update the query parameters with the correct start and
        // end dates
        log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates " + dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());
        // reset the dates in the configuration, no need to reset then in
        // the Query settings object
        config.setBeginDate(dateIndexData.getBeginDate());
        config.setEndDate(dateIndexData.getEndDate());
    } else {
        log.info("Date index not needed for this query");
    }
    return queryTree;
}

Example 13 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method loadQueryIterator.

protected Future<IteratorSetting> loadQueryIterator(final MetadataHelper metadataHelper, final ShardQueryConfiguration config, final Query settings, final String queryString, final Boolean isFullTable) throws DatawaveQueryException {
    return builderThread.submit(() -> {
        // VersioningIterator is typically set at 20 on the table
        IteratorSetting cfg = new IteratorSetting(config.getBaseIteratorPriority() + 40, "query", getQueryIteratorClass());
        addOption(cfg, Constants.RETURN_TYPE, config.getReturnType().toString(), false);
        addOption(cfg, QueryOptions.FULL_TABLE_SCAN_ONLY, Boolean.toString(isFullTable), false);
        if (sourceLimit > 0) {
            addOption(cfg, QueryOptions.LIMIT_SOURCES, Long.toString(sourceLimit), false);
        }
        if (config.getCollectTimingDetails()) {
            addOption(cfg, QueryOptions.COLLECT_TIMING_DETAILS, Boolean.toString(true), false);
        }
        if (config.getSendTimingToStatsd()) {
            addOption(cfg, QueryOptions.STATSD_HOST_COLON_PORT, config.getStatsdHost() + ':' + Integer.toString(config.getStatsdPort()), false);
            addOption(cfg, QueryOptions.STATSD_MAX_QUEUE_SIZE, Integer.toString(config.getStatsdMaxQueueSize()), false);
        }
        if (config.getHdfsSiteConfigURLs() != null) {
            addOption(cfg, QueryOptions.HDFS_SITE_CONFIG_URLS, config.getHdfsSiteConfigURLs(), false);
        }
        if (config.getHdfsFileCompressionCodec() != null) {
            addOption(cfg, QueryOptions.HDFS_FILE_COMPRESSION_CODEC, config.getHdfsFileCompressionCodec(), false);
        }
        if (config.getZookeeperConfig() != null) {
            addOption(cfg, QueryOptions.ZOOKEEPER_CONFIG, config.getZookeeperConfig(), false);
        }
        if (config.getIvaratorCacheDirConfigs() != null && !config.getIvaratorCacheDirConfigs().isEmpty()) {
            addOption(cfg, QueryOptions.IVARATOR_CACHE_DIR_CONFIG, IvaratorCacheDirConfig.toJson(getShuffledIvaratoCacheDirConfigs(config)), false);
        }
        addOption(cfg, QueryOptions.IVARATOR_CACHE_BUFFER_SIZE, Integer.toString(config.getIvaratorCacheBufferSize()), false);
        addOption(cfg, QueryOptions.IVARATOR_SCAN_PERSIST_THRESHOLD, Long.toString(config.getIvaratorCacheScanPersistThreshold()), false);
        addOption(cfg, QueryOptions.IVARATOR_SCAN_TIMEOUT, Long.toString(config.getIvaratorCacheScanTimeout()), false);
        addOption(cfg, QueryOptions.COLLECT_TIMING_DETAILS, Boolean.toString(config.getCollectTimingDetails()), false);
        addOption(cfg, QueryOptions.MAX_INDEX_RANGE_SPLIT, Integer.toString(config.getMaxFieldIndexRangeSplit()), false);
        addOption(cfg, QueryOptions.MAX_IVARATOR_OPEN_FILES, Integer.toString(config.getIvaratorMaxOpenFiles()), false);
        addOption(cfg, QueryOptions.MAX_IVARATOR_RESULTS, Long.toString(config.getMaxIvaratorResults()), false);
        addOption(cfg, QueryOptions.IVARATOR_NUM_RETRIES, Integer.toString(config.getIvaratorNumRetries()), false);
        addOption(cfg, QueryOptions.IVARATOR_PERSIST_VERIFY, Boolean.toString(config.isIvaratorPersistVerify()), false);
        addOption(cfg, QueryOptions.IVARATOR_PERSIST_VERIFY_COUNT, Integer.toString(config.getIvaratorPersistVerifyCount()), false);
        addOption(cfg, QueryOptions.MAX_EVALUATION_PIPELINES, Integer.toString(config.getMaxEvaluationPipelines()), false);
        addOption(cfg, QueryOptions.MAX_PIPELINE_CACHED_RESULTS, Integer.toString(config.getMaxPipelineCachedResults()), false);
        addOption(cfg, QueryOptions.MAX_IVARATOR_SOURCES, Integer.toString(config.getMaxIvaratorSources()), false);
        if (config.getYieldThresholdMs() != Long.MAX_VALUE && config.getYieldThresholdMs() > 0) {
            addOption(cfg, QueryOptions.YIELD_THRESHOLD_MS, Long.toString(config.getYieldThresholdMs()), false);
        }
        addOption(cfg, QueryOptions.SORTED_UIDS, Boolean.toString(config.isSortedUIDs()), false);
        configureTypeMappings(config, cfg, metadataHelper, compressMappings);
        configureAdditionalOptions(config, cfg);
        try {
            addOption(cfg, QueryOptions.INDEX_ONLY_FIELDS, QueryOptions.buildFieldStringFromSet(metadataHelper.getIndexOnlyFields(config.getDatatypeFilter())), true);
            addOption(cfg, QueryOptions.COMPOSITE_FIELDS, QueryOptions.buildFieldStringFromSet(metadataHelper.getCompositeToFieldMap(config.getDatatypeFilter()).keySet()), true);
            addOption(cfg, QueryOptions.INDEXED_FIELDS, QueryOptions.buildFieldStringFromSet(metadataHelper.getIndexedFields(config.getDatatypeFilter())), true);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_RETRIEVAL_ERROR, e);
            throw new DatawaveQueryException(qe);
        }
        try {
            CompositeMetadata compositeMetadata = metadataHelper.getCompositeMetadata().filter(config.getQueryFieldsDatatypes().keySet());
            if (compositeMetadata != null && !compositeMetadata.isEmpty())
                addOption(cfg, QueryOptions.COMPOSITE_METADATA, java.util.Base64.getEncoder().encodeToString(CompositeMetadata.toBytes(compositeMetadata)), false);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.COMPOSITE_METADATA_CONFIG_ERROR, e);
            throw new DatawaveQueryException(qe);
        }
        String datatypeFilter = config.getDatatypeFilterAsString();
        addOption(cfg, QueryOptions.DATATYPE_FILTER, datatypeFilter, false);
        try {
            addOption(cfg, QueryOptions.CONTENT_EXPANSION_FIELDS, Joiner.on(',').join(metadataHelper.getContentFields(config.getDatatypeFilter())), false);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.CONTENT_FIELDS_RETRIEVAL_ERROR, e);
            throw new DatawaveQueryException(qe);
        }
        if (config.isDebugMultithreadedSources()) {
            addOption(cfg, QueryOptions.DEBUG_MULTITHREADED_SOURCES, Boolean.toString(config.isDebugMultithreadedSources()), false);
        }
        if (config.isLimitFieldsPreQueryEvaluation()) {
            addOption(cfg, QueryOptions.LIMIT_FIELDS_PRE_QUERY_EVALUATION, Boolean.toString(config.isLimitFieldsPreQueryEvaluation()), false);
        }
        if (config.getLimitFieldsField() != null) {
            addOption(cfg, QueryOptions.LIMIT_FIELDS_FIELD, config.getLimitFieldsField(), false);
        }
        return cfg;
    });
}

Example 14 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method timedFixQueryPropertyMarkers.

protected ASTJexlScript timedFixQueryPropertyMarkers(QueryStopwatch timers, ASTJexlScript script) throws DatawaveQueryException {
    // Fix query property markers with multiple sources.
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - fix query property markers with multiple sources");
    try {
        script = QueryPropertyMarkerSourceConsolidator.consolidate(script);
    } catch (Exception e) {
        throw new DatawaveQueryException("Failed to fix query property markers with multiple sources", e);
    }
    if (log.isDebugEnabled()) {
        logQuery(script, "Query after fixing query property markers with multiple sources");
    }
    stopwatch.stop();
    return script;
}

Example 15 with DatawaveQueryException

use of datawave.query.exceptions.DatawaveQueryException in project datawave by NationalSecurityAgency.

the class FacetedQueryPlanner method getQueryIterator.

@Override
public IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, Query settings, String queryString, Boolean isFullTable) throws DatawaveQueryException {
    if (isFullTable) {
        QueryException qe = new QueryException(DatawaveErrorCode.FULL_TABLE_SCAN_DISALLOWED);
        throw new FullTableScansDisallowedException(qe);
    }
    IteratorSetting cfg = super.getQueryIterator(metadataHelper, config, settings, queryString, isFullTable);
    if (!usePrecomputedFacets)
        cfg.setIteratorClass(DynamicFacetIterator.class.getName());
    else {
        config.setShardTableName(facetedConfig.getFacetTableName());
        cfg.setIteratorClass(FacetedTableIterator.class.getName());
    }
    cfg.addOption(DynamicFacetIterator.FACETED_SEARCH_TYPE, facetedConfig.getType().toString());
    cfg.addOption(DynamicFacetIterator.FACETED_MINIMUM, Integer.toString(facetedConfig.getMinimumFacetCount()));
    cfg.addOption(DynamicFacetIterator.FACETED_SEARCH_FIELDS, Joiner.on(",").join(facetedConfig.getFacetedFields()));
    if (log.isTraceEnabled())
        log.trace("Configuration is " + facetedConfig);
    return cfg;
}