use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testTheSimplestOfQueries.
@Test
public void testTheSimplestOfQueries() throws Exception {
String originalQuery = "FOO == 'bag'";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Set<Range> expectedRanges = Sets.newHashSet(makeTestRange("20190314", "datatype1\u0000234"), makeTestRange("20190314", "datatype1\u0000345"));
for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range from expected ranges: " + range.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testIntersection_NestedUnionOfHighCardinalityTerm_withSeek.
// (A && (B || C))
@Test
public void testIntersection_NestedUnionOfHighCardinalityTerm_withSeek() throws Exception {
String originalQuery = "(FOO == 'lowest_card' && (FOO == 'high_card' || FOO == 'highest_card'))";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
config.setBeginDate(sdf.parse("20190310"));
config.setEndDate(sdf.parse("20190320"));
config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
// Create expected ranges verbosely, so it is obvious which shards contribute to the results.
Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
Range range2 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
Range range3 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
rangeStream.setLimitScanners(true);
CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
for (QueryPlan queryPlan : queryPlans) {
Iterable<Range> ranges = queryPlan.getRanges();
for (Range range : ranges) {
assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testNegatedIndexWithResultsStandaloneNot.
@Test
public void testNegatedIndexWithResultsStandaloneNot() throws Exception {
String originalQuery = "FOO == 'bag' && !(FOO == 'bar')";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Range range1 = makeTestRange("20190314", "datatype1\u0000234");
Range range2 = makeTestRange("20190314", "datatype1\u0000345");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testNonIndexedNumeric.
@Test
public void testNonIndexedNumeric() throws Exception {
String originalQuery = "(FOO == 'bag' && BAR == 4)";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
helper.addFields(Lists.newArrayList("BAR"));
Range range1 = makeTestRange("20190314", "datatype1\u0000234");
Range range2 = makeTestRange("20190314", "datatype1\u0000345");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testPrune.
@Test
public void testPrune() throws Exception {
String originalQuery = "FOO=='bag' || FOO=='qwertylikeskeyboards'";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("BAR", Sets.newHashSet(new NoOpType()));
dataTypes.putAll("BAR", Sets.newHashSet(new NumberType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
helper.addFields(Lists.newArrayList("BAR"));
Range range1 = makeTestRange("20190314", "datatype1\u0000234");
Range range2 = makeTestRange("20190314", "datatype1\u0000345");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
assertEquals("FOO == 'bag'", JexlStringBuildingVisitor.buildQuery(queryPlan.getQueryTree()));
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Aggregations