Search in sources :

Example 6 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_HighAndLowCardinality_withSeek.

// (A && B)
@Test
public void testIntersection_HighAndLowCardinality_withSeek() throws Exception {
    String originalQuery = "(FOO == 'lowest_card' && FOO == 'highest_card')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Create expected ranges verbosely, so it is obvious which shards contribute to the results.
    Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
    Range range2 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
    Range range3 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) SimpleDateFormat(java.text.SimpleDateFormat) Test(org.junit.Test)

Example 7 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testNegatedIndexWithNoResultsStandaloneNot.

@Test
public void testNegatedIndexWithNoResultsStandaloneNot() throws Exception {
    String originalQuery = "FOO == 'bag' && !(FOO == 'tacocat')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) Test(org.junit.Test)

Example 8 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testAndOneFieldIndexed.

@Test
public void testAndOneFieldIndexed() throws Exception {
    String originalQuery = "(FOO == 'bag' && TACO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(ImmutableSet.of("TACO"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 9 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testNegatedIndexWithResults.

@Test
public void testNegatedIndexWithResults() throws Exception {
    String originalQuery = "FOO == 'bag' && BAR != 'bar'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) Test(org.junit.Test)

Example 10 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testUnIndexedNumericAsString.

@Test
public void testUnIndexedNumericAsString() throws Exception {
    String originalQuery = "(FOO == 'bag' && BAR == '+aE1')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(Lists.newArrayList("BAR"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) Test(org.junit.Test)

Aggregations

LcNoDiacriticsType (datawave.data.type.LcNoDiacriticsType)49 Type (datawave.data.type.Type)46 NumberType (datawave.data.type.NumberType)44 Test (org.junit.Test)42 NoOpType (datawave.data.type.NoOpType)41 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)36 MockMetadataHelper (datawave.query.util.MockMetadataHelper)35 ScannerFactory (datawave.query.tables.ScannerFactory)34 RangeFactoryForTests.makeTestRange (datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange)25 QueryPlan (datawave.query.planner.QueryPlan)25 Range (org.apache.accumulo.core.data.Range)25 LcType (datawave.data.type.LcType)9 SimpleDateFormat (java.text.SimpleDateFormat)9 IpAddressType (datawave.data.type.IpAddressType)6 JexlASTHelper (datawave.query.jexl.JexlASTHelper)2 Date (java.util.Date)2 Set (java.util.Set)2 ASTReference (org.apache.commons.jexl2.parser.ASTReference)2 JexlNode (org.apache.commons.jexl2.parser.JexlNode)2 ParseException (org.apache.commons.jexl2.parser.ParseException)2