Examples with DefaultQueryPlanner datawave.query.planner.DefaultQueryPlanner used on opensource projects

Example 1 with DefaultQueryPlanner

use of datawave.query.planner.DefaultQueryPlanner in project datawave by NationalSecurityAgency.

the class AnyFieldQueryTest method testRegexPushdownAnyfield.

@Test
public void testRegexPushdownAnyfield() throws Exception {
    String roPhrase = RE_OP + "'ro.*'";
    String anyRo = this.dataManager.convertAnyField(roPhrase);
    String oPhrase = RE_OP + "'.*a'";
    String anyO = this.dataManager.convertAnyField(oPhrase);
    String query = Constants.ANY_FIELD + roPhrase + AND_OP + Constants.ANY_FIELD + oPhrase;
    RegexPushdownTransformRule rule = new RegexPushdownTransformRule();
    rule.setRegexPatterns(Arrays.asList("\\.\\*[0-9a-zA-Z]", "[0-9a-zA-Z]\\.\\*"));
    ((DefaultQueryPlanner) (logic.getQueryPlanner())).setTransformRules(Collections.singletonList(rule));
    // Test the plan with all expansions
    try {
        String plan = getPlan(query, true, true);
        fail("Expected failure for regex pushdown on anyfield");
    } catch (Exception e) {
    // expected
    }
    try {
        String plan = getPlan(query, false, true);
        fail("Expected failure for regex pushdown on anyfield");
    } catch (Exception e) {
    // expected
    }
    try {
        String plan = getPlan(query, true, false);
        fail("Expected failure for regex pushdown on anyfield");
    } catch (Exception e) {
    // expected
    }
    // test running the query
    String expect = anyRo + AND_OP + anyO;
    try {
        runTest(query, expect);
        fail("Expected failure for regex pushdown on anyfield");
    } catch (Exception e) {
    // expected
    }
}

Example 2 with DefaultQueryPlanner

use of datawave.query.planner.DefaultQueryPlanner in project datawave by NationalSecurityAgency.

the class MixedGeoAndGeoWaveTest method getShardQueryLogic.

private ShardQueryLogic getShardQueryLogic() {
    ShardQueryLogic logic = new ShardQueryLogic(this.logic);
    // increase the depth threshold
    logic.setMaxDepthThreshold(20);
    // set the pushdown threshold really high to avoid collapsing uids into shards (overrides setCollapseUids if #terms is greater than this threshold)
    ((DefaultQueryPlanner) (logic.getQueryPlanner())).setPushdownThreshold(1000000);
    URL hdfsSiteConfig = this.getClass().getResource("/testhadoop.config");
    logic.setHdfsSiteConfigURLs(hdfsSiteConfig.toExternalForm());
    setupIvarator(logic);
    return logic;
}

Example 3 with DefaultQueryPlanner

use of datawave.query.planner.DefaultQueryPlanner in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method initialize.

public void initialize(ShardQueryConfiguration config, Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    // Set the connector and the authorizations into the config object
    config.setConnector(connection);
    config.setAuthorizations(auths);
    config.setMaxScannerBatchSize(getMaxScannerBatchSize());
    config.setMaxIndexBatchSize(getMaxIndexBatchSize());
    setScannerFactory(new ScannerFactory(config));
    // load params before parsing jexl string so these can be injected
    loadQueryParameters(config, settings);
    String jexlQueryString = getJexlQueryString(settings);
    if (null == jexlQueryString) {
        throw new IllegalArgumentException("Query cannot be null");
    } else {
        config.setQueryString(jexlQueryString);
    }
    final Date beginDate = settings.getBeginDate();
    if (null == beginDate) {
        throw new IllegalArgumentException("Begin date cannot be null");
    } else {
        config.setBeginDate(beginDate);
    }
    final Date endDate = settings.getEndDate();
    if (null == endDate) {
        throw new IllegalArgumentException("End date cannot be null");
    } else {
        config.setEndDate(endDate);
    }
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths, config.isRawTypes());
    DateIndexHelper dateIndexHelper = prepareDateIndexHelper(connection, this.getDateIndexTableName(), auths);
    if (config.isDateIndexTimeTravel()) {
        dateIndexHelper.setTimeTravel(config.isDateIndexTimeTravel());
    }
    QueryPlanner queryPlanner = getQueryPlanner();
    if (queryPlanner instanceof DefaultQueryPlanner) {
        DefaultQueryPlanner currentQueryPlanner = (DefaultQueryPlanner) queryPlanner;
        currentQueryPlanner.setMetadataHelper(metadataHelper);
        currentQueryPlanner.setDateIndexHelper(dateIndexHelper);
        QueryModelProvider queryModelProvider = currentQueryPlanner.getQueryModelProviderFactory().createQueryModelProvider();
        if (queryModelProvider instanceof MetadataHelperQueryModelProvider) {
            ((MetadataHelperQueryModelProvider) queryModelProvider).setMetadataHelper(metadataHelper);
            ((MetadataHelperQueryModelProvider) queryModelProvider).setConfig(config);
        }
        if (null != queryModelProvider.getQueryModel()) {
            queryModel = queryModelProvider.getQueryModel();
        }
    }
    if (this.queryModel == null)
        loadQueryModel(metadataHelper, config);
    getQueryPlanner().setCreateUidsIteratorClass(createUidsIteratorClass);
    getQueryPlanner().setUidIntersector(uidIntersector);
    validateConfiguration(config);
    if (getCardinalityConfiguration() != null && (!config.getBlacklistedFields().isEmpty() || !config.getProjectFields().isEmpty())) {
        // Ensure that fields used for resultCardinalities are returned. They will be removed in the DocumentTransformer.
        // Modify the projectFields and blacklistFields only for this stage, then return to the original values.
        // Not advisable to create a copy of the config object due to the embedded timers.
        Set<String> originalBlacklistedFields = new HashSet<>(config.getBlacklistedFields());
        Set<String> originalProjectFields = new HashSet<>(config.getProjectFields());
        // this will be caught when loadQueryParameters is called
        if (!config.getBlacklistedFields().isEmpty()) {
            config.setBlacklistedFields(getCardinalityConfiguration().getRevisedBlacklistFields(queryModel, originalBlacklistedFields));
        }
        if (!config.getProjectFields().isEmpty()) {
            config.setProjectFields(getCardinalityConfiguration().getRevisedProjectFields(queryModel, originalProjectFields));
        }
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
        config.setBlacklistedFields(originalBlacklistedFields);
        config.setProjectFields(originalProjectFields);
    } else {
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
    }
    TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("ShardQueryLogic - Get iterator of queries");
    if (this.queries != null) {
        config.setQueries(this.queries.iterator());
    }
    config.setQueryString(getQueryPlanner().getPlannedScript());
    stopwatch.stop();
}

Example 4 with DefaultQueryPlanner

use of datawave.query.planner.DefaultQueryPlanner in project datawave by NationalSecurityAgency.

the class ContentFunctionQueryTest method getShardQueryLogic.

private ShardQueryLogic getShardQueryLogic(boolean useIvarator) {
    ShardQueryLogic logic = new ShardQueryLogic(this.logic);
    // increase the depth threshold
    logic.setMaxDepthThreshold(20);
    // set the pushdown threshold really high to avoid collapsing uids into shards (overrides setCollapseUids if #terms is greater than this threshold)
    ((DefaultQueryPlanner) (logic.getQueryPlanner())).setPushdownThreshold(1000000);
    URL hdfsSiteConfig = this.getClass().getResource("/testhadoop.config");
    logic.setHdfsSiteConfigURLs(hdfsSiteConfig.toExternalForm());
    logic.setIvaratorCacheDirConfigs(ivaratorCacheDirConfigs);
    if (useIvarator)
        setupIvarator(logic);
    return logic;
}

Example 5 with DefaultQueryPlanner

use of datawave.query.planner.DefaultQueryPlanner in project datawave by NationalSecurityAgency.

the class UseOccurrenceToCountInJexlContextTest method setup.

@Before
public void setup() throws Exception {
    TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
    logic = new ShardQueryLogic();
    logic.setMetadataTableName(QueryTestTableHelper.METADATA_TABLE_NAME);
    logic.setTableName(TableName.SHARD);
    logic.setIndexTableName(TableName.SHARD_INDEX);
    logic.setReverseIndexTableName(TableName.SHARD_RINDEX);
    logic.setMaxResults(5000);
    logic.setMaxWork(25000);
    logic.setModelTableName(QueryTestTableHelper.METADATA_TABLE_NAME);
    logic.setQueryPlanner(new DefaultQueryPlanner());
    logic.setIncludeGroupingContext(true);
    logic.setMarkingFunctions(new MarkingFunctions.Default());
    logic.setMetadataHelperFactory(new MetadataHelperFactory());
    logic.setDateIndexHelperFactory(new DateIndexHelperFactory());
    logic.setMaxEvaluationPipelines(1);
    deserializer = new KryoDocumentDeserializer();
}