Search in sources :

Example 51 with MockMetadataHelper

use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testNonIndexedNumeric.

@Test
public void testNonIndexedNumeric() throws Exception {
    String originalQuery = "(FOO == 'bag' && BAR == 4)";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(Lists.newArrayList("BAR"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) Test(org.junit.Test)

Example 52 with MockMetadataHelper

use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testPrune.

@Test
public void testPrune() throws Exception {
    String originalQuery = "FOO=='bag' || FOO=='qwertylikeskeyboards'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("BAR", Sets.newHashSet(new NoOpType()));
    dataTypes.putAll("BAR", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(Lists.newArrayList("BAR"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        assertEquals("FOO == 'bag'", JexlStringBuildingVisitor.buildQuery(queryPlan.getQueryTree()));
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) NoOpType(datawave.data.type.NoOpType) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 53 with MockMetadataHelper

use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testShardAndDaysHints6.

@Test
public void testShardAndDaysHints6() throws Exception {
    String originalQuery = "(FOO == 'oreo') && ((filter:include(FOO, 'tardy') && (SHARDS_AND_DAYS = '20190312,20190313,20190314')) || (filter:include(FOO, 'bardy') && (SHARDS_AND_DAYS = '20190312,20190313,20190314')) )";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Set<Range> expectedRanges = Sets.newHashSet();
    for (String shard : Lists.newArrayList("20190314_1")) {
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000123"));
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000345"));
    }
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        // verify the query plan dropped no terms
        JexlNode queryTree = JexlASTHelper.parseJexlQuery(queryPlan.getQueryString());
        JexlNode expectedTree = JexlASTHelper.parseJexlQuery("(((SHARDS_AND_DAYS = '20190314') && filter:include(FOO, 'tardy')) || ((SHARDS_AND_DAYS = '20190314') && filter:include(FOO, 'bardy'))) && FOO == 'oreo'");
        JexlNodeAssert.assertThat(queryTree).isEqualTo(expectedTree);
        // verify the range
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue(expectedRanges.size() + " expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) QueryPlan(datawave.query.planner.QueryPlan) ScannerFactory(datawave.query.tables.ScannerFactory) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) JexlNode(org.apache.commons.jexl2.parser.JexlNode) Test(org.junit.Test)

Example 54 with MockMetadataHelper

use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testAndNoFieldIndexed.

@Test
public void testAndNoFieldIndexed() throws Exception {
    String originalQuery = "(TACO == 'bag' && TACO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    assertFalse(new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script).iterator().hasNext());
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ScannerFactory(datawave.query.tables.ScannerFactory) Test(org.junit.Test)

Example 55 with MockMetadataHelper

use of datawave.query.util.MockMetadataHelper in project datawave by NationalSecurityAgency.

the class RangeStreamTestX method runTest.

private void runTest(String query, List<Range> expectedRanges, List<String> expectedQueries) throws Exception {
    assertEquals("Expected ranges and queries do not match, ranges: " + expectedRanges.size() + " queries: " + expectedQueries.size(), expectedRanges.size(), expectedQueries.size());
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(query);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    // config.setBeginDate(new Date(0));
    config.setBeginDate(sdf.parse("20200101"));
    config.setEndDate(sdf.parse("20200105"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("A", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("B", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("C", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("D", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    config.setShardsPerDayThreshold(2);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Run a standard limited-scanner range stream.
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    runTest(rangeStream, script, expectedRanges, expectedQueries);
    // Run a default range stream.
    rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(false);
    runTest(rangeStream, script, expectedRanges, expectedQueries);
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ScannerFactory(datawave.query.tables.ScannerFactory) SimpleDateFormat(java.text.SimpleDateFormat)

Aggregations

MockMetadataHelper (datawave.query.util.MockMetadataHelper)55 Test (org.junit.Test)45 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)40 LcNoDiacriticsType (datawave.data.type.LcNoDiacriticsType)35 Type (datawave.data.type.Type)35 NoOpType (datawave.data.type.NoOpType)34 ScannerFactory (datawave.query.tables.ScannerFactory)34 NumberType (datawave.data.type.NumberType)33 RangeFactoryForTests.makeTestRange (datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange)25 QueryPlan (datawave.query.planner.QueryPlan)25 Range (org.apache.accumulo.core.data.Range)25 SimpleDateFormat (java.text.SimpleDateFormat)9 ShardQueryConfiguration (datawave.query.config.ShardQueryConfiguration)7 Before (org.junit.Before)4 JexlNode (org.apache.commons.jexl2.parser.JexlNode)3 JexlArgumentDescriptor (datawave.query.jexl.functions.arguments.JexlArgumentDescriptor)2 MockDateIndexHelper (datawave.query.util.MockDateIndexHelper)2 ASTFunctionNode (org.apache.commons.jexl2.parser.ASTFunctionNode)2 ArrayListMultimap (com.google.common.collect.ArrayListMultimap)1 Multimap (com.google.common.collect.Multimap)1