Examples with QueryModel datawave.query.model.QueryModel used on opensource projects

Example 11 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method updateQueryTree.

protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, QueryData queryData, Query settings) throws DatawaveQueryException {
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Parse query");
    config.setQueryTree(parseQueryAndValidatePattern(query, stopwatch));
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Query after initial parse:");
    }
    stopwatch.stop();
    Map<String, String> optionsMap = new HashMap<>();
    if (query.contains(QueryFunctions.QUERY_FUNCTION_NAMESPACE + ':')) {
        // only do the extra tree visit if the function is present
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - parse out queryOptions from options function");
        config.setQueryTree(QueryOptionsFromQueryVisitor.collect(config.getQueryTree(), optionsMap));
        if (!optionsMap.isEmpty()) {
            QueryOptionsSwitch.apply(optionsMap, config);
        }
        stopwatch.stop();
    }
    // groom the query so that any nodes with the literal on the left and the identifier on
    // the right will be re-ordered to simplify subsequent processing
    config.setQueryTree(timedInvertSwappedNodes(timers, config.getQueryTree()));
    config.setQueryTree(timedFixNotNullIntent(timers, config.getQueryTree()));
    config.setQueryTree(timedIncludeDateFilters(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, dateIndexHelper, settings));
    // note this must be called after we do the date adjustments per the query date type in addDateFilters
    timedCapDateRange(timers, config);
    // Find unmarked bounded ranges
    if (UnmarkedBoundedRangeDetectionVisitor.findUnmarkedBoundedRanges(config.getQueryTree())) {
        throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
    }
    if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
        config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
    } else {
        // look for the shards and days hint in the query settings
        // the shards and days hint cannot always be specified in the query string when using certain query parsers
        Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
        if (StringUtils.isNotBlank(parameter.getParameterValue())) {
            optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
            config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
        }
    }
    // extract #NO_EXPANSION function, if it exists
    config.setQueryTree(parseNoExpansionFields(timers, config.getQueryTree(), config));
    // flatten the tree
    config.setQueryTree(timedFlatten(timers, config.getQueryTree()));
    validateQuerySize("initial parse", config.getQueryTree(), config);
    config.setQueryTree(timedApplyRules(timers, config.getQueryTree(), config, metadataHelper, scannerFactory));
    config.setQueryTree(timedFixNegativeNumbers(timers, config.getQueryTree()));
    // Fix any query property markers that have multiple unwrapped sources.
    config.setQueryTree(timedFixQueryPropertyMarkers(timers, config.getQueryTree()));
    // Ensure that all ASTIdentifier nodes (field names) are upper-case to be consistent with what is enforced at ingest time
    config.setQueryTree(timedUpperCaseIdentifiers(timers, config.getQueryTree(), config, metadataHelper));
    config.setQueryTree(timedRewriteNegations(timers, config.getQueryTree()));
    QueryModel queryModel = loadQueryModel(config);
    config.setQueryTree(timedApplyQueryModel(timers, config.getQueryTree(), config, metadataHelper, queryModel));
    // Enforce unique terms within an AND or OR expression.
    if (config.getEnforceUniqueTermsWithinExpressions()) {
        config.setQueryTree(timedEnforceUniqueTermsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique AND'd terms within OR expressions.
    if (config.getEnforceUniqueConjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueConjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique OR'd terms within AND expressions.
    if (config.getEnforceUniqueDisjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueDisjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    Set<String> indexOnlyFields = loadIndexedFields(config);
    if (disableBoundedLookup) {
        // LT,GT or ER node, we should expand it
        if (BoundedRangeDetectionVisitor.mustExpandBoundedRange(config, metadataHelper, config.getQueryTree()))
            disableBoundedLookup = false;
    }
    if (!indexOnlyFields.isEmpty()) {
        config.setQueryTree(expandRegexFunctionNodes(config.getQueryTree(), config, metadataHelper, indexOnlyFields));
    }
    config.setQueryTree(processTree(config.getQueryTree(), config, settings, metadataHelper, scannerFactory, queryData, timers, queryModel));
    // ExpandCompositeTerms was here
    boolean containsIndexOnlyFields = false;
    if (!indexOnlyFields.isEmpty() && !disableBoundedLookup) {
        // Figure out if the query contained any index only terms so we know
        // if we have to force it down the field-index path with event-specific
        // ranges
        containsIndexOnlyFields = timedCheckForIndexOnlyFieldsInQuery(timers, "Check for Index-Only Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper, indexOnlyFields);
    }
    boolean containsComposites = timedCheckForCompositeFields(timers, "Check for Composite Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    boolean sortedUIDS = timedCheckForSortedUids(timers, "Check for Sorted UIDs", config.getQueryTree(), config);
    // check the query for any fields that are term frequencies
    // if any exist, populate the shard query config with these fields
    timedCheckForTokenizedFields(timers, "Check for term frequency (tokenized) fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query Final", config.getQueryTree()));
    }
    return config.getQueryTree();
}

Example 12 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class WiredQueryModelProvider method setQueryModel.

// public void setQueryModel(QueryModel queryModel) {
// this.queryModel = queryModel;
// }
public void setQueryModel(Map<String, String> mappings) {
    QueryModel model = new QueryModel();
    for (Map.Entry<String, String> mapping : mappings.entrySet()) {
        for (String value : StringUtils.split(mapping.getValue(), ',')) {
            model.addTermToModel(mapping.getKey(), value);
            model.addTermToReverseModel(value, mapping.getKey());
        }
    }
    this.queryModel = model;
}

Example 13 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class MixedGeoAndGeoWaveTest method getResultsIterator.

private Iterator getResultsIterator(String queryString, ShardQueryLogic logic) throws Exception {
    MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
    params.putSingle(QUERY_LOGIC_NAME, "EventQuery");
    params.putSingle(QUERY_STRING, queryString);
    params.putSingle(QUERY_NAME, "geoQuery");
    params.putSingle(QUERY_PERSISTENCE, "PERSISTENT");
    params.putSingle(QUERY_AUTHORIZATIONS, AUTHS);
    params.putSingle(QUERY_EXPIRATION, "20200101 000000.000");
    params.putSingle(QUERY_BEGIN, BEGIN_DATE);
    params.putSingle(QUERY_END, END_DATE);
    QueryParameters queryParams = new QueryParametersImpl();
    queryParams.validate(params);
    Set<Authorizations> auths = new HashSet<>();
    auths.add(new Authorizations(AUTHS));
    Query query = new QueryImpl();
    query.initialize(USER, Arrays.asList(USER_DN), null, queryParams, null);
    ShardQueryConfiguration config = ShardQueryConfiguration.create(logic, query);
    QueryModel queryModel = new QueryModel();
    queryModel.addTermToModel(GEO_FIELD, GEO_FIELD);
    queryModel.addTermToModel(GEO_FIELD, POINT_FIELD);
    config.setQueryModel(queryModel);
    logic.initialize(config, instance.getConnector("root", PASSWORD), query, auths);
    logic.setupQuery(config);
    return logic.getTransformIterator(query);
}