Examples with PreNormalizedAttribute datawave.query.attributes.PreNormalizedAttribute used on opensource projects

Example 6 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws ParseException, Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<String>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<String>();
    Set<Document> docs = new HashSet<Document>();
    for (Map.Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.trace(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID");
        if (attr == null)
            attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
}

Example 7 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class TermFrequencyIndexIteratorTest method testScanFullRangeTLD.

@Test
public void testScanFullRangeTLD() throws IOException {
    Range r = new Range(getFiKey("row", "type1", "123.345.456", "FOO", "alf"), true, getFiKey("row", "type1", "123.345.456.2", "FOO", "buz"), true);
    aggregator = new TLDTermFrequencyAggregator(fieldsToKeep, filter, -1);
    TermFrequencyIndexIterator iterator = new TermFrequencyIndexIterator(r, source, null, typeMetadata, true, null, aggregator);
    // jump to the first doc
    iterator.seek(null, null, true);
    Assert.assertTrue(iterator.hasTop());
    Document d = iterator.document();
    Assert.assertTrue(d != null);
    Assert.assertTrue(d.getDictionary().size() == 2);
    Assert.assertTrue(d.getDictionary().get("FOO") != null);
    Assert.assertTrue(d.getDictionary().get("RECORD_ID") != null);
    Assert.assertTrue(d.getDictionary().get("FOO").getData() != null);
    Assert.assertTrue(((Set) d.getDictionary().get("FOO").getData()).size() == 6);
    Iterator<PreNormalizedAttribute> i = ((Set) d.getDictionary().get("FOO").getData()).iterator();
    Assert.assertTrue(i.next().getValue().equals("bar"));
    Assert.assertTrue(i.next().getValue().equals("baz"));
    Assert.assertTrue(i.next().getValue().equals("buf"));
    Assert.assertTrue(i.next().getValue().equals("buz"));
    Assert.assertTrue(i.next().getValue().equals("alf"));
    Assert.assertTrue(i.next().getValue().equals("arm"));
    iterator.next();
    Assert.assertFalse(iterator.hasTop());
}

Example 8 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class TermFrequencyIndexIteratorTest method testScanFullRangeExclusiveTLD.

@Test
public void testScanFullRangeExclusiveTLD() throws IOException, ParseException {
    Range r = new Range(getFiKey("row", "type1", "123.345.456", "FOO", "alf"), false, getFiKey("row", "type1", "123.345.456.2", "FOO", "buz"), false);
    filter = new TLDEventDataFilter(JexlASTHelper.parseJexlQuery("FOO=='bar' || FOO=='baz' || FOO=='buf' || FOO=='arm'"), typeMetadata, null, null, -1, -1, Collections.emptyMap(), null, fieldsToKeep);
    aggregator = new TLDTermFrequencyAggregator(fieldsToKeep, filter, -1);
    TermFrequencyIndexIterator iterator = new TermFrequencyIndexIterator(r, source, null, typeMetadata, true, null, aggregator);
    // jump to the first doc
    iterator.seek(null, null, true);
    Assert.assertTrue(iterator.hasTop());
    Document d = iterator.document();
    Assert.assertTrue(d != null);
    Assert.assertTrue(d.getDictionary().size() == 2);
    Assert.assertTrue(d.getDictionary().get("FOO") != null);
    Assert.assertTrue(d.getDictionary().get("RECORD_ID") != null);
    Assert.assertTrue(d.getDictionary().get("FOO").getData() != null);
    Iterator<PreNormalizedAttribute> i = ((Set) d.getDictionary().get("FOO").getData()).iterator();
    Assert.assertTrue(i.next().getValue().equals("bar"));
    Assert.assertTrue(i.next().getValue().equals("baz"));
    Assert.assertTrue(i.next().getValue().equals("buf"));
    Assert.assertTrue(i.next().getValue().equals("arm"));
    iterator.next();
    Assert.assertFalse(iterator.hasTop());
}

Example 9 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class TermFrequencyIndexIteratorTest method testScanFullRangeExclusiveEventDataQueryExpressionFilter.

@Test
public void testScanFullRangeExclusiveEventDataQueryExpressionFilter() throws IOException, ParseException {
    Range r = new Range(getFiKey("row", "type1", "123.345.456", "FOO", "alf"), false, getFiKey("row", "type1", "123.345.456.2", "FOO", "buz"), false);
    filter = new EventDataQueryExpressionFilter(JexlASTHelper.parseJexlQuery("FOO=='bar' || FOO=='baz' || FOO=='buf' || FOO=='arm'"), typeMetadata, fieldsToKeep);
    aggregator = new TLDTermFrequencyAggregator(fieldsToKeep, filter, -1);
    TermFrequencyIndexIterator iterator = new TermFrequencyIndexIterator(r, source, null, typeMetadata, true, null, aggregator);
    // jump to the first doc
    iterator.seek(null, null, true);
    Assert.assertTrue(iterator.hasTop());
    Document d = iterator.document();
    Assert.assertTrue(d != null);
    Assert.assertTrue(d.getDictionary().size() == 2);
    Assert.assertTrue(d.getDictionary().get("FOO") != null);
    Assert.assertTrue(d.getDictionary().get("RECORD_ID") != null);
    Assert.assertTrue(d.getDictionary().get("FOO").getData() != null);
    Iterator<PreNormalizedAttribute> i = ((Set) d.getDictionary().get("FOO").getData()).iterator();
    Assert.assertTrue(i.next().getValue().equals("bar"));
    Assert.assertTrue(i.next().getValue().equals("baz"));
    Assert.assertTrue(i.next().getValue().equals("buf"));
    Assert.assertTrue(i.next().getValue().equals("buz"));
    Assert.assertTrue(i.next().getValue().equals("alf"));
    Assert.assertTrue(i.next().getValue().equals("arm"));
    iterator.next();
    Assert.assertFalse(iterator.hasTop());
}

Example 10 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class EvaluationPhaseFilterFunctionsTest method toNonTypedValueTuple.

private static ValueTuple toNonTypedValueTuple(String csv) {
    String[] tokens = csv.split(",");
    String field = tokens[0];
    String normalized = tokens[2];
    PreNormalizedAttribute attribute = new PreNormalizedAttribute(normalized, null, false);
    return new ValueTuple(field, "second", normalized, attribute);
}