Examples with QueryPlan datawave.query.planner.QueryPlan used on opensource projects

Example 21 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class PushdownFunction method redistributeQueries.

protected void redistributeQueries(Multimap<String, QueryPlan> serverPlan, TabletLocator tl, QueryPlan currentPlan) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    List<Range> ranges = Lists.newArrayList(currentPlan.getRanges());
    if (!ranges.isEmpty()) {
        Map<String, Map<KeyExtent, List<Range>>> binnedRanges = binRanges(tl, config.getConnector().getInstance(), ranges);
        for (String server : binnedRanges.keySet()) {
            Map<KeyExtent, List<Range>> hostedExtentMap = binnedRanges.get(server);
            Iterable<Range> rangeIter = Lists.newArrayList();
            for (Entry<KeyExtent, List<Range>> rangeEntry : hostedExtentMap.entrySet()) {
                if (log.isTraceEnabled())
                    log.trace("Adding range from " + rangeEntry.getValue());
                rangeIter = Iterables.concat(rangeIter, rangeEntry.getValue());
            }
            if (log.isTraceEnabled())
                log.trace("Adding query tree " + JexlStringBuildingVisitor.buildQuery(currentPlan.getQueryTree()) + " " + currentPlan.getSettings().size() + " for " + server);
            serverPlan.put(server, new QueryPlan(currentPlan.getQueryTree(), rangeIter, currentPlan.getSettings(), currentPlan.getColumnFamilies()));
        }
    }
}

Example 22 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testNegatedNonIndexed.

@Test
public void testNegatedNonIndexed() throws Exception {
    String originalQuery = "FOO == 'bag' && TACO != 'tacocat'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 23 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_OfTwoNestedUnions_LeftLowCardTerms_withSeek.

// (A || B) && (C || D)
@Test
public void testIntersection_OfTwoNestedUnions_LeftLowCardTerms_withSeek() throws Exception {
    String originalQuery = "(FOO == 'low_card' || FOO == 'lowest_card') && (FOO == 'high_card' || FOO == 'highest_card')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Create expected ranges verbosely, so it is obvious which shards contribute to the results.
    Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
    Range range2 = makeTestRange("20190312_1", "datatype1\u0000a.b.c");
    Range range3 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
    Range range4 = makeTestRange("20190315_33", "datatype1\u0000a.b.c");
    Range range5 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
    Range range6 = makeTestRange("20190317_1", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 24 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testShardAndDaysHints.

public void testShardAndDaysHints(String originalQuery) throws Exception {
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Set<Range> expectedRanges = Sets.newHashSet();
    for (String shard : Lists.newArrayList("20190314_0", "20190314_1", "20190314_9")) {
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000345"));
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000456"));
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000567"));
    }
    for (String shard : Lists.newArrayList("20190314_10", "20190314_100")) {
        expectedRanges.add(makeTestRange(shard, "datatype2\u0000345"));
        expectedRanges.add(makeTestRange(shard, "datatype2\u0000456"));
        expectedRanges.add(makeTestRange(shard, "datatype2\u0000567"));
    }
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue(expectedRanges.size() + " expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 25 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersectionOfTwoUnions.

/**
 * Interesting case when the right side terms are indexed and the left side terms are not indexed.
 *
 * @throws Exception
 */
@Test
public void testIntersectionOfTwoUnions() throws Exception {
    String originalQuery = "(FOO == 'bag' || FOO == 'bar') && (TACO == 'shell' || TACO == 'tacocat')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(ImmutableSet.of("TACO"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    // streamPlans(script) to populate the StreamContext.
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}