Examples with LcNoDiacriticsType datawave.data.type.LcNoDiacriticsType used on opensource projects

Example 46 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testPrune.

@Test
public void testPrune() throws Exception {
    String originalQuery = "FOO=='bag' || FOO=='qwertylikeskeyboards'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("BAR", Sets.newHashSet(new NoOpType()));
    dataTypes.putAll("BAR", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(Lists.newArrayList("BAR"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        assertEquals("FOO == 'bag'", JexlStringBuildingVisitor.buildQuery(queryPlan.getQueryTree()));
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 47 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testShardAndDaysHints6.

@Test
public void testShardAndDaysHints6() throws Exception {
    String originalQuery = "(FOO == 'oreo') && ((filter:include(FOO, 'tardy') && (SHARDS_AND_DAYS = '20190312,20190313,20190314')) || (filter:include(FOO, 'bardy') && (SHARDS_AND_DAYS = '20190312,20190313,20190314')) )";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Set<Range> expectedRanges = Sets.newHashSet();
    for (String shard : Lists.newArrayList("20190314_1")) {
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000123"));
        expectedRanges.add(makeTestRange(shard, "datatype1\u0000345"));
    }
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        // verify the query plan dropped no terms
        JexlNode queryTree = JexlASTHelper.parseJexlQuery(queryPlan.getQueryString());
        JexlNode expectedTree = JexlASTHelper.parseJexlQuery("(((SHARDS_AND_DAYS = '20190314') && filter:include(FOO, 'tardy')) || ((SHARDS_AND_DAYS = '20190314') && filter:include(FOO, 'bardy'))) && FOO == 'oreo'");
        JexlNodeAssert.assertThat(queryTree).isEqualTo(expectedTree);
        // verify the range
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue(expectedRanges.size() + " expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 48 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testAndNoFieldIndexed.

@Test
public void testAndNoFieldIndexed() throws Exception {
    String originalQuery = "(TACO == 'bag' && TACO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    assertFalse(new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script).iterator().hasNext());
}

Example 49 with LcNoDiacriticsType

use of datawave.data.type.LcNoDiacriticsType in project datawave by NationalSecurityAgency.

the class RangeStreamTestX method runTest.

private void runTest(String query, List<Range> expectedRanges, List<String> expectedQueries) throws Exception {
    assertEquals("Expected ranges and queries do not match, ranges: " + expectedRanges.size() + " queries: " + expectedQueries.size(), expectedRanges.size(), expectedQueries.size());
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(query);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    // config.setBeginDate(new Date(0));
    config.setBeginDate(sdf.parse("20200101"));
    config.setEndDate(sdf.parse("20200105"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("A", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("B", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("C", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("D", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    config.setShardsPerDayThreshold(2);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    // Run a standard limited-scanner range stream.
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    runTest(rangeStream, script, expectedRanges, expectedQueries);
    // Run a default range stream.
    rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(false);
    runTest(rangeStream, script, expectedRanges, expectedQueries);
}