Search in sources :

Example 11 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testUnion_HighCardWithNestedIntersectionOfLowCardTerms_withSeek.

// A || (B && C)
@Test
public void testUnion_HighCardWithNestedIntersectionOfLowCardTerms_withSeek() throws Exception {
    String originalQuery = "(FOO == 'low_card' || (FOO == 'high_card' && FOO == 'highest_card'))";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Union with the Intersection of the high & highest cardinality terms means we hit every 'high_card' shard.
    Set<Range> expectedRanges = new HashSet<>();
    for (int day = 0; day < 8; day += 2) {
        for (int ii = 1; ii < 50; ii++) {
            expectedRanges.add(makeTestRange("2019031" + day + "_" + ii, "datatype1\u0000a.b.c"));
            expectedRanges.add(makeTestRange("2019031" + day + "_" + ii, "datatype1\u0000d.e.f"));
        }
    }
    expectedRanges.add(makeTestRange("20190315_33", "datatype1\u0000a.b.c"));
    expectedRanges.add(makeTestRange("20190317_1", "datatype1\u0000a.b.c"));
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) SimpleDateFormat(java.text.SimpleDateFormat) Test(org.junit.Test)

Example 12 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testShardAndDaysHint.

@Test
public void testShardAndDaysHint() throws Exception {
    String originalQuery = "(FOO == 'bardy') && (SHARDS_AND_DAYS = '20190314_2,20190314_1')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314_1", "datatype1\u0000345");
    Range range2 = makeTestRange("20190314_1", "datatype1\u0000456");
    Range range3 = makeTestRange("20190314_1", "datatype1\u0000567");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) QueryPlan(datawave.query.planner.QueryPlan) ScannerFactory(datawave.query.tables.ScannerFactory) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 13 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testDropTwoPredicates.

@Test
public void testDropTwoPredicates() throws Exception {
    String originalQuery = "LAUGH == 'bahahaha' && ( FOO == 'boohoo' || FOO == 'idontexist' || FOO == 'neitherdoi!' )";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190314"));
    config.setEndDate(sdf.parse("20190315"));
    config.setShardsPerDayThreshold(3);
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(Arrays.asList("FOO", "LAUGH"));
    // Create expected ranges verbosely, so it is obvious which shards contribute to the results.
    Range range1 = makeTestRange("20190314_0", "datatype1\u0000345");
    Range range2 = makeTestRange("20190314_0", "datatype1\u0000456");
    Range range3 = makeTestRange("20190314_0", "datatype1\u0000567");
    Range range4 = makeTestRange("20190314_0", "datatype1\u00001345");
    Range range5 = makeTestRange("20190314_0", "datatype1\u00002456");
    Range range6 = makeTestRange("20190314_0", "datatype1\u00003567");
    Set<Range> shard0 = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    range1 = makeTestRange("20190314_1", "datatype1\u0000345");
    range2 = makeTestRange("20190314_1", "datatype1\u0000456");
    range3 = makeTestRange("20190314_1", "datatype1\u0000567");
    range4 = makeTestRange("20190314_1", "datatype1\u00001345");
    range5 = makeTestRange("20190314_1", "datatype1\u00002456");
    range6 = makeTestRange("20190314_1", "datatype1\u00003567");
    Set<Range> shard1 = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    range1 = makeTestRange("20190314_10", "datatype2\u0000345");
    range2 = makeTestRange("20190314_10", "datatype2\u0000456");
    range3 = makeTestRange("20190314_10", "datatype2\u0000567");
    range4 = makeTestRange("20190314_10", "datatype2\u00001345");
    range5 = makeTestRange("20190314_10", "datatype2\u00002456");
    range6 = makeTestRange("20190314_10", "datatype2\u00003567");
    Set<Range> shard10 = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    range1 = makeTestRange("20190314_100", "datatype2\u0000345");
    range2 = makeTestRange("20190314_100", "datatype2\u0000456");
    range3 = makeTestRange("20190314_100", "datatype2\u0000567");
    range4 = makeTestRange("20190314_100", "datatype2\u00001345");
    range5 = makeTestRange("20190314_100", "datatype2\u00002456");
    range6 = makeTestRange("20190314_100", "datatype2\u00003567");
    Set<Range> shard100 = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    range1 = makeTestRange("20190314_9", "datatype1\u0000345");
    range2 = makeTestRange("20190314_9", "datatype1\u0000456");
    range3 = makeTestRange("20190314_9", "datatype1\u0000567");
    range4 = makeTestRange("20190314_9", "datatype1\u00001345");
    range5 = makeTestRange("20190314_9", "datatype1\u00002456");
    range6 = makeTestRange("20190314_9", "datatype1\u00003567");
    Set<Range> shard9 = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    Set<Range> expectedRanges = Sets.newHashSet();
    expectedRanges.addAll(shard0);
    expectedRanges.addAll(shard1);
    expectedRanges.addAll(shard10);
    expectedRanges.addAll(shard100);
    expectedRanges.addAll(shard9);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    // streamPlans(script) to populate the StreamContext.
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) SimpleDateFormat(java.text.SimpleDateFormat) Test(org.junit.Test)

Example 14 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testOrBothIndexed.

@Test
public void testOrBothIndexed() throws Exception {
    String originalQuery = "(FOO == 'bag' || FOO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Range range3 = makeTestRange("20190314", "datatype1\u0000123");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) QueryPlan(datawave.query.planner.QueryPlan) ScannerFactory(datawave.query.tables.ScannerFactory) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 15 with QueryPlan

use of datawave.query.planner.QueryPlan in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testUnion_OfTwoNestedIntersections_LeftLowCardTerms_withSeek.

// (A && B) || (C && D)
@Test
public void testUnion_OfTwoNestedIntersections_LeftLowCardTerms_withSeek() throws Exception {
    String originalQuery = "(FOO == 'low_card' && FOO == 'lowest_card') || (FOO == 'high_card' && FOO == 'highest_card')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Set<Range> expectedRanges = new HashSet<>();
    for (int day = 0; day < 8; day += 2) {
        for (int ii = 1; ii < 50; ii++) {
            expectedRanges.add(makeTestRange("2019031" + day + "_" + ii, "datatype1\u0000a.b.c"));
            expectedRanges.add(makeTestRange("2019031" + day + "_" + ii, "datatype1\u0000d.e.f"));
        }
    }
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) SimpleDateFormat(java.text.SimpleDateFormat) Test(org.junit.Test)

Aggregations

QueryPlan (datawave.query.planner.QueryPlan)32 Range (org.apache.accumulo.core.data.Range)31 RangeFactoryForTests.makeTestRange (datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange)26 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)26 LcNoDiacriticsType (datawave.data.type.LcNoDiacriticsType)25 NoOpType (datawave.data.type.NoOpType)25 NumberType (datawave.data.type.NumberType)25 Type (datawave.data.type.Type)25 ScannerFactory (datawave.query.tables.ScannerFactory)25 MockMetadataHelper (datawave.query.util.MockMetadataHelper)25 Test (org.junit.Test)24 SimpleDateFormat (java.text.SimpleDateFormat)8 JexlNode (org.apache.commons.jexl2.parser.JexlNode)3 ArrayList (java.util.ArrayList)2 RangeFactoryForTests.makeDayRange (datawave.common.test.utils.query.RangeFactoryForTests.makeDayRange)1 RangeFactoryForTests.makeShardedRange (datawave.common.test.utils.query.RangeFactoryForTests.makeShardedRange)1 JexlASTHelper (datawave.query.jexl.JexlASTHelper)1 SessionOptions (datawave.query.tables.SessionOptions)1 ScannerChunk (datawave.query.tables.async.ScannerChunk)1 HashMap (java.util.HashMap)1