Examples with DateIndexHelper datawave.query.util.DateIndexHelper used on opensource projects

Example 1 with DateIndexHelper

use of datawave.query.util.DateIndexHelper in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method initialize.

public void initialize(ShardQueryConfiguration config, Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    // Set the connector and the authorizations into the config object
    config.setConnector(connection);
    config.setAuthorizations(auths);
    config.setMaxScannerBatchSize(getMaxScannerBatchSize());
    config.setMaxIndexBatchSize(getMaxIndexBatchSize());
    setScannerFactory(new ScannerFactory(config));
    // load params before parsing jexl string so these can be injected
    loadQueryParameters(config, settings);
    String jexlQueryString = getJexlQueryString(settings);
    if (null == jexlQueryString) {
        throw new IllegalArgumentException("Query cannot be null");
    } else {
        config.setQueryString(jexlQueryString);
    }
    final Date beginDate = settings.getBeginDate();
    if (null == beginDate) {
        throw new IllegalArgumentException("Begin date cannot be null");
    } else {
        config.setBeginDate(beginDate);
    }
    final Date endDate = settings.getEndDate();
    if (null == endDate) {
        throw new IllegalArgumentException("End date cannot be null");
    } else {
        config.setEndDate(endDate);
    }
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths, config.isRawTypes());
    DateIndexHelper dateIndexHelper = prepareDateIndexHelper(connection, this.getDateIndexTableName(), auths);
    if (config.isDateIndexTimeTravel()) {
        dateIndexHelper.setTimeTravel(config.isDateIndexTimeTravel());
    }
    QueryPlanner queryPlanner = getQueryPlanner();
    if (queryPlanner instanceof DefaultQueryPlanner) {
        DefaultQueryPlanner currentQueryPlanner = (DefaultQueryPlanner) queryPlanner;
        currentQueryPlanner.setMetadataHelper(metadataHelper);
        currentQueryPlanner.setDateIndexHelper(dateIndexHelper);
        QueryModelProvider queryModelProvider = currentQueryPlanner.getQueryModelProviderFactory().createQueryModelProvider();
        if (queryModelProvider instanceof MetadataHelperQueryModelProvider) {
            ((MetadataHelperQueryModelProvider) queryModelProvider).setMetadataHelper(metadataHelper);
            ((MetadataHelperQueryModelProvider) queryModelProvider).setConfig(config);
        }
        if (null != queryModelProvider.getQueryModel()) {
            queryModel = queryModelProvider.getQueryModel();
        }
    }
    if (this.queryModel == null)
        loadQueryModel(metadataHelper, config);
    getQueryPlanner().setCreateUidsIteratorClass(createUidsIteratorClass);
    getQueryPlanner().setUidIntersector(uidIntersector);
    validateConfiguration(config);
    if (getCardinalityConfiguration() != null && (!config.getBlacklistedFields().isEmpty() || !config.getProjectFields().isEmpty())) {
        // Ensure that fields used for resultCardinalities are returned. They will be removed in the DocumentTransformer.
        // Modify the projectFields and blacklistFields only for this stage, then return to the original values.
        // Not advisable to create a copy of the config object due to the embedded timers.
        Set<String> originalBlacklistedFields = new HashSet<>(config.getBlacklistedFields());
        Set<String> originalProjectFields = new HashSet<>(config.getProjectFields());
        // this will be caught when loadQueryParameters is called
        if (!config.getBlacklistedFields().isEmpty()) {
            config.setBlacklistedFields(getCardinalityConfiguration().getRevisedBlacklistFields(queryModel, originalBlacklistedFields));
        }
        if (!config.getProjectFields().isEmpty()) {
            config.setProjectFields(getCardinalityConfiguration().getRevisedProjectFields(queryModel, originalProjectFields));
        }
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
        config.setBlacklistedFields(originalBlacklistedFields);
        config.setProjectFields(originalProjectFields);
    } else {
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
    }
    TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("ShardQueryLogic - Get iterator of queries");
    if (this.queries != null) {
        config.setQueries(this.queries.iterator());
    }
    config.setQueryString(getQueryPlanner().getPlannedScript());
    stopwatch.stop();
}

Example 2 with DateIndexHelper

use of datawave.query.util.DateIndexHelper in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method addDateFilters.

/**
 * Adding date filters if the query parameters specify that the dates are to be other than the default
 *
 * @param queryTree
 * @param scannerFactory
 * @param metadataHelper
 * @param config
 * @return the updated query tree
 * @throws TableNotFoundException
 */
public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, Query settings) throws TableNotFoundException, DatawaveQueryException {
    String defaultDateType = config.getDefaultDateTypeName();
    String dateType = defaultDateType;
    Parameter dateTypeParameter = settings.findParameter(QueryParameters.DATE_RANGE_TYPE);
    if (dateTypeParameter != null && dateTypeParameter.getParameterValue() != null) {
        String parm = dateTypeParameter.getParameterValue().trim();
        if (!parm.isEmpty()) {
            dateType = parm.toUpperCase();
        }
    }
    // time, then we need to modify the query
    if (!dateType.equals(defaultDateType)) {
        log.info("Using the date index for " + dateType);
        // if no date index helper configured, then we are in error
        if (dateIndexHelper == null) {
            throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
        }
        // get all of the fields used for this date type
        DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(), config.getDatatypeFilter());
        if (dateIndexData.getFields().isEmpty()) {
            log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
            // If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
            // Also it will be understandable from the plan as to why no results were returned.
            dateIndexData.getFields().add(Constants.NO_FIELD);
        }
        log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
        // now for each field, add an expression to filter that date
        List<JexlNode> andChildren = new ArrayList<>();
        for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
            andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
        }
        List<JexlNode> orChildren = new ArrayList<>();
        for (String field : dateIndexData.getFields()) {
            orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
        }
        if (orChildren.size() > 1) {
            andChildren.add(JexlNodeFactory.createOrNode(orChildren));
        } else {
            andChildren.addAll(orChildren);
        }
        JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
        JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));
        // now lets update the query parameters with the correct start and
        // end dates
        log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates " + dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());
        // reset the dates in the configuration, no need to reset then in
        // the Query settings object
        config.setBeginDate(dateIndexData.getBeginDate());
        config.setEndDate(dateIndexData.getEndDate());
    } else {
        log.info("Date index not needed for this query");
    }
    return queryTree;
}