use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.
the class JexlASTHelperTest method testFindIndexedRange.
@Test
public void testFindIndexedRange() throws Exception {
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(Collections.singleton("A"));
ASTJexlScript script = JexlASTHelper.parseJexlQuery("((_Bounded_ = true) && (A < 'b' && A > 'a'))");
LiteralRange range = JexlASTHelper.findRange().indexedOnly(null, helper).getRange(script.jjtGetChild(0));
Assert.assertNotNull(range);
Assert.assertNotNull(range.getLowerNode());
Assert.assertNotNull(range.getUpperNode());
Assert.assertEquals("a", range.getLower());
Assert.assertEquals("b", range.getUpper());
Assert.assertFalse(range.isLowerInclusive());
Assert.assertFalse(range.isUpperInclusive());
script = JexlASTHelper.parseJexlQuery("B < 5 && B > 1");
range = JexlASTHelper.findRange().indexedOnly(null, helper).getRange(script.jjtGetChild(0));
Assert.assertNull(range);
}
use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.
the class NodeTransformVisitorTest method testPushdown.
private void testPushdown(String original, String expected, List<NodeTransformRule> rules) throws Exception {
// create a query tree
ASTJexlScript originalScript = JexlASTHelper.parseJexlQuery(original);
MockMetadataHelper helper = new MockMetadataHelper();
// apply the visitor
ASTJexlScript resultScript = NodeTransformVisitor.transform(originalScript, rules, new ShardQueryConfiguration(), helper);
// Verify the script is as expected, and has a valid lineage.
assertScriptEquality(resultScript, expected);
assertLineage(resultScript);
// Verify the original script was not modified, and still has a valid lineage.
assertScriptEquality(originalScript, original);
assertLineage(originalScript);
}
use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testBothIndexedPrune.
@Test
public void testBothIndexedPrune() throws Exception {
String originalQuery = "(FOO == 'barter' || FOO == 'baggy')";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Range range1 = makeTestRange("20190314_1", "datatype1\u0000123");
Range range2 = makeTestRange("20190314_1", "datatype1\u0000345");
Range range3 = makeTestRange("20190414_1", "datatype1\u0000123");
Range range4 = makeTestRange("20190414_1", "datatype1\u0000345");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testIntersection_NestedUnionOfLowCardinalityTerm_withSeek.
// A && (B || C)
@Test
public void testIntersection_NestedUnionOfLowCardinalityTerm_withSeek() throws Exception {
String originalQuery = "(FOO == 'highest_card' && (FOO == 'low_card' || FOO == 'lowest_card'))";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
config.setBeginDate(sdf.parse("20190310"));
config.setEndDate(sdf.parse("20190320"));
config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
Range range2 = makeTestRange("20190312_1", "datatype1\u0000a.b.c");
Range range3 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
Range range4 = makeTestRange("20190315_33", "datatype1\u0000a.b.c");
Range range5 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
Range range6 = makeTestRange("20190317_1", "datatype1\u0000a.b.c");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
rangeStream.setLimitScanners(true);
CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
for (QueryPlan queryPlan : queryPlans) {
Iterable<Range> ranges = queryPlan.getRanges();
for (Range range : ranges) {
assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testIntersection_ofDayRangesAndShardRange.
// A && B when A term is day ranges and B term is a single shard range within the last day.
@Test
public void testIntersection_ofDayRangesAndShardRange() throws Exception {
String originalQuery = "FOO == 'day_ranges' && FOO == 'shard_range'";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
// config.setBeginDate(new Date(0));
config.setBeginDate(sdf.parse("20190310"));
config.setEndDate(sdf.parse("20190320"));
config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
config.setShardsPerDayThreshold(2);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
// Create expected ranges verbosely, so it is obvious which shards contribute to the results.
Range range1 = makeTestRange("20190310_21", "datatype1\u0000a.b.c");
// Fun story. It's hard to roll up to a day range when you seek most of the way through the day and don't have all the shards for the day.
// Range range2 = makeTestRange("20190315_51", "datatype1\u0000a.b.c");
Set<Range> expectedRanges = Sets.newHashSet(range1);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
rangeStream.setLimitScanners(true);
CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
for (QueryPlan queryPlan : queryPlans) {
Iterable<Range> ranges = queryPlan.getRanges();
for (Range range : ranges) {
assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Aggregations