Examples with PreNormalizedAttribute datawave.query.attributes.PreNormalizedAttribute used on opensource projects

Example 11 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class FunctionalSetTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    logic.setMaxEvaluationPipelines(1);
    logic.setMaxDepthThreshold(6);
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID for query:" + querystr + "  " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records for query:" + querystr, expected.size(), resultSet.size());
}

Example 12 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class IfThisTestFailsThenHitTermsAreBroken method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Multimap<String, String> expectedHitTerms) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    settings.setParameters(extraParms);
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> uuidAttr = (TypeAttribute<?>) attr;
        String uuid = uuidAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + uuid, expected.contains(uuid));
        Attribute<?> hitTermAttribute = d.get(JexlEvaluation.HIT_TERM_FIELD);
        if (hitTermAttribute instanceof Attributes) {
            Attributes hitTerms = (Attributes) hitTermAttribute;
            for (Attribute<?> hitTerm : hitTerms.getAttributes()) {
                log.debug("hitTerm:" + hitTerm);
                String hitString = hitTerm.getData().toString();
                log.debug("as string:" + hitString);
                log.debug("expectedHitTerms:" + expectedHitTerms);
                Assert.assertNotEquals(hitTerm.getTimestamp(), Long.MAX_VALUE);
                // make sure this hitString is in the map, and remove it
                boolean result = expectedHitTerms.get(uuid).remove(hitString);
                if (result == false) {
                    log.debug("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                    Assert.fail("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                } else {
                    log.debug("removed hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                }
            }
        } else if (hitTermAttribute instanceof Attribute) {
            log.debug("hitTerm:" + (Attribute<?>) hitTermAttribute);
            String hitString = ((Attribute<?>) hitTermAttribute).getData().toString();
            log.debug("as string:" + hitString);
            log.debug("expectedHitTerms:" + expectedHitTerms);
            boolean result = expectedHitTerms.get(uuid).remove(hitString);
            if (result == false) {
                log.debug("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms);
                Assert.fail("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms);
            } else {
                log.debug("removed hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerm:" + (Attribute<?>) hitTermAttribute);
            }
        }
        resultSet.add(uuid);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
    // the map is empty if there were no unexpected hit terms in it
    log.debug("expectedHitTerms:" + expectedHitTerms);
    Assert.assertTrue(expectedHitTerms.isEmpty());
}

Example 13 with PreNormalizedAttribute

use of datawave.query.attributes.PreNormalizedAttribute in project datawave by NationalSecurityAgency.

the class UseOccurrenceToCountInJexlContextTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Multimap<String, String> expectedHitTerms, Connector connector) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    settings.setParameters(Collections.singletonMap("hit.list", "true"));
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> uuidAttr = (TypeAttribute<?>) attr;
        String uuid = uuidAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + uuid, expected.contains(uuid));
        Attribute<?> hitTermAttribute = d.get(JexlEvaluation.HIT_TERM_FIELD);
        if (hitTermAttribute instanceof Attributes) {
            Attributes hitTerms = (Attributes) hitTermAttribute;
            for (Attribute<?> hitTerm : hitTerms.getAttributes()) {
                log.debug("hitTerm:" + hitTerm);
                String hitString = hitTerm.getData().toString();
                log.debug("as string:" + hitString);
                log.debug("expectedHitTerms:" + expectedHitTerms);
                // make sure this hitString is in the map, and remove it
                boolean result = expectedHitTerms.get(uuid).remove(hitString);
                if (result == false) {
                    log.debug("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                    Assert.fail("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                } else {
                    log.debug("removed hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerms:" + hitTerms);
                }
            }
        } else if (hitTermAttribute instanceof Attribute) {
            log.debug("hitTerm:" + (Attribute<?>) hitTermAttribute);
            String hitString = ((Attribute<?>) hitTermAttribute).getData().toString();
            log.debug("as string:" + hitString);
            log.debug("expectedHitTerms:" + expectedHitTerms);
            boolean result = expectedHitTerms.get(uuid).remove(hitString);
            if (result == false) {
                log.debug("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms);
                Assert.fail("failed to find hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms);
            } else {
                log.debug("removed hitString:" + hitString + " for uuid:" + uuid + " in expectedHitTerms:" + expectedHitTerms + " from hitTerm:" + (Attribute<?>) hitTermAttribute);
            }
        }
        resultSet.add(uuid);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
    // the map is empty if there were no unexpected hit terms in it
    log.debug("expectedHitTerms:" + expectedHitTerms);
    Assert.assertTrue(expectedHitTerms.isEmpty());
}