Examples with QueryPlan datawave.query.planner.QueryPlan used on opensource projects

Example 1 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class PushdownFunction method apply.

public List<ScannerChunk> apply(QueryData qd) {
    Multimap<String, QueryPlan> serverPlan = ArrayListMultimap.create();
    List<ScannerChunk> chunks = Lists.newArrayList();
    try {
        redistributeQueries(serverPlan, tl, new QueryPlan(qd));
        for (String server : serverPlan.keySet()) {
            Collection<QueryPlan> plans = serverPlan.get(server);
            Set<QueryPlan> reducedSet = Sets.newHashSet(plans);
            for (QueryPlan plan : reducedSet) {
                Integer hashCode = plan.hashCode();
                if (queryPlanSet.contains(hashCode)) {
                    continue;
                } else
                    queryPlanSet.clear();
                queryPlanSet.add(hashCode);
                try {
                    SessionOptions options = new SessionOptions();
                    if (log.isTraceEnabled()) {
                        log.trace("setting ranges" + plan.getRanges());
                        log.trace("range set size" + plan.getSettings().size());
                    }
                    for (IteratorSetting setting : plan.getSettings()) {
                        options.addScanIterator(setting);
                    }
                    for (IteratorSetting setting : customSettings) {
                        options.addScanIterator(setting);
                    }
                    for (String cf : plan.getColumnFamilies()) {
                        options.fetchColumnFamily(new Text(cf));
                    }
                    options.setQueryConfig(this.config);
                    chunks.add(new ScannerChunk(options, Lists.newArrayList(plan.getRanges()), server));
                } catch (Exception e) {
                    log.error(e);
                    throw new AccumuloException(e);
                }
            }
        }
    } catch (AccumuloException e) {
        throw new RuntimeException(e);
    } catch (AccumuloSecurityException e) {
        throw new RuntimeException(e);
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    } catch (ParseException e) {
        throw new RuntimeException(e);
    }
    return chunks;
}

Example 2 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testBothIndexedPrune.

@Test
public void testBothIndexedPrune() throws Exception {
    String originalQuery = "(FOO == 'barter' || FOO == 'baggy')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314_1", "datatype1\u0000123");
    Range range2 = makeTestRange("20190314_1", "datatype1\u0000345");
    Range range3 = makeTestRange("20190414_1", "datatype1\u0000123");
    Range range4 = makeTestRange("20190414_1", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 3 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_NestedUnionOfLowCardinalityTerm_withSeek.

// A && (B || C)
@Test
public void testIntersection_NestedUnionOfLowCardinalityTerm_withSeek() throws Exception {
    String originalQuery = "(FOO == 'highest_card' && (FOO == 'low_card' || FOO == 'lowest_card'))";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
    Range range2 = makeTestRange("20190312_1", "datatype1\u0000a.b.c");
    Range range3 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
    Range range4 = makeTestRange("20190315_33", "datatype1\u0000a.b.c");
    Range range5 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
    Range range6 = makeTestRange("20190317_1", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 4 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_ofDayRangesAndShardRange.

// A && B when A term is day ranges and B term is a single shard range within the last day.
@Test
public void testIntersection_ofDayRangesAndShardRange() throws Exception {
    String originalQuery = "FOO == 'day_ranges' && FOO == 'shard_range'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    // config.setBeginDate(new Date(0));
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    config.setShardsPerDayThreshold(2);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Create expected ranges verbosely, so it is obvious which shards contribute to the results.
    Range range1 = makeTestRange("20190310_21", "datatype1\u0000a.b.c");
    // Fun story. It's hard to roll up to a day range when you seek most of the way through the day and don't have all the shards for the day.
    // Range range2 = makeTestRange("20190315_51", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 5 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_HighAndLowCardinality_withSeek.

// (A && B)
@Test
public void testIntersection_HighAndLowCardinality_withSeek() throws Exception {
    String originalQuery = "(FOO == 'lowest_card' && FOO == 'highest_card')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Create expected ranges verbosely, so it is obvious which shards contribute to the results.
    Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
    Range range2 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
    Range range3 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}