Examples with TypeAttribute datawave.query.attributes.TypeAttribute used on opensource projects

Example 1 with TypeAttribute

use of datawave.query.attributes.TypeAttribute in project datawave by NationalSecurityAgency.

the class GroupingTransform method flatten.

/**
 * <pre>
 * flush used the countingMap:
 * [[MALE, 16],
 * [MALE, 20],
 * [40, MALE],
 * [40, MALE],
 * [MALE, 22] x 2,
 * [FEMALE, 18],
 * [MALE, 24],
 * [20, MALE],
 * [30, MALE],
 * [FEMALE, 18],
 * [34, MALE]]
 *
 * to create documents list: [
 * {AGE=16, COUNT=1, GENDER=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {COUNT=1, ETA=20, GENERE=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {COUNT=1, ETA=40, GENERE=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {AGE=40, COUNT=1, GENDER=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {COUNT=2, ETA=22, GENERE=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {AGE=18, COUNT=1, GENDER=FEMALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {COUNT=1, ETA=24, GENERE=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {AGE=20, COUNT=1, GENDER=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {AGE=30, COUNT=1, GENDER=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {COUNT=1, ETA=18, GENERE=FEMALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false,
 * {AGE=34, COUNT=1, GENDER=MALE}:20130101_0 test%00;-d5uxna.msizfm.-oxy0iu: [ALL] 1356998400000 false]
 *
 * which is then flattened to just one document with the fields and counts correlated with a grouping context suffix:
 *
 * {
 * AGE.0=16, GENDER.0=MALE, COUNT.0=1,
 * ETA.1=20, GENERE.1=MALE, COUNT.1=1,
 * ETA.2=40, GENERE.2=MALE, COUNT.2=1,
 * AGE.3=40, GENDER.3=MALE, COUNT.3=1,
 * ETA.4=22, GENERE.4=MALE, COUNT.4=2,
 * AGE.5=18, GENDER.5=FEMALE, COUNT.5=1,
 * ETA.6=24, GENERE.6=MALE, COUNT.6=1,
 * AGE.7=20, GENDER.7=MALE, COUNT.7=1,
 * AGE.8=30, GENDER.8=MALE, COUNT.8=1,
 * ETA.9=18, GENERE.9=FEMALE, COUNT.9=1,
 * AGE.A=34, GENDER.A=MALE, COUNT.A=1,
 * }
 * </pre>
 *
 * The Attributes, which have had their visibilities merged, are copied into normal TypeAttributes for serialization to the webserver.
 *
 * @param documents
 */
private void flatten(List<Document> documents) {
    log.trace("flatten {}", documents);
    Document theDocument = new Document(documents.get(documents.size() - 1).getMetadata(), true);
    int context = 0;
    Set<ColumnVisibility> visibilities = new HashSet<>();
    for (Document document : documents) {
        log.trace("document: {}", document);
        for (Entry<String, Attribute<? extends Comparable<?>>> entry : document.entrySet()) {
            String name = entry.getKey();
            visibilities.add(entry.getValue().getColumnVisibility());
            Attribute<? extends Comparable<?>> attribute = entry.getValue();
            attribute.setColumnVisibility(entry.getValue().getColumnVisibility());
            // call copy() on the GroupingTypeAttribute to get a plain TypeAttribute
            // instead of a GroupingTypeAttribute that is package protected and won't serialize
            theDocument.put(name + "." + Integer.toHexString(context).toUpperCase(), (TypeAttribute) attribute.copy(), true, false);
        }
        context++;
    }
    ColumnVisibility combinedVisibility = combine(visibilities);
    log.trace("combined visibilities: {} to {}", visibilities, combinedVisibility);
    theDocument.setColumnVisibility(combinedVisibility);
    documents.clear();
    log.trace("flattened document: {}", theDocument);
    documents.add(theDocument);
}

Example 2 with TypeAttribute

use of datawave.query.attributes.TypeAttribute in project datawave by NationalSecurityAgency.

the class QueryLogicTestHarness method assertLogicResults.

// =============================================
// assert methods
/**
 * Determines if the correct results were obtained for a query.
 *
 * @param logic
 *            key/value response data
 * @param expected
 *            list of key values expected within response data
 * @param checkers
 *            list of additional validation methods
 */
public void assertLogicResults(BaseQueryLogic<Map.Entry<Key, Value>> logic, Collection<String> expected, List<DocumentChecker> checkers) {
    Set<String> actualResults = new HashSet<>();
    if (log.isDebugEnabled()) {
        log.debug("    ======  expected id(s)  ======");
        for (String e : expected) {
            log.debug("id(" + e + ")");
        }
    }
    for (Map.Entry<Key, Value> entry : logic) {
        if (FinalDocumentTrackingIterator.isFinalDocumentKey(entry.getKey())) {
            continue;
        }
        final Document document = this.deserializer.apply(entry).getValue();
        // check all of the types to ensure that all are keepers as defined in the
        // AttributeFactory class
        int count = 0;
        for (Attribute<? extends Comparable<?>> attribute : document.getAttributes()) {
            if (attribute instanceof TimingMetadata) {
            // ignore
            } else if (attribute instanceof Attributes) {
                Attributes attrs = (Attributes) attribute;
                Collection<Class<?>> types = new HashSet<>();
                for (Attribute<? extends Comparable<?>> attr : attrs.getAttributes()) {
                    count++;
                    if (attr instanceof TypeAttribute) {
                        Type<? extends Comparable<?>> type = ((TypeAttribute<?>) attr).getType();
                        if (Objects.nonNull(type)) {
                            types.add(type.getClass());
                        }
                    }
                }
                Assert.assertEquals(AttributeFactory.getKeepers(types), types);
            } else {
                count++;
            }
        }
        // ignore empty documents (possible when only passing FinalDocument back)
        if (count == 0) {
            continue;
        }
        // parse the document
        String extractedResult = this.parser.parse(entry.getKey(), document);
        log.debug("result(" + extractedResult + ") key(" + entry.getKey() + ") document(" + document + ")");
        // verify expected results
        Assert.assertNotNull("extracted result", extractedResult);
        Assert.assertFalse("duplicate result(" + extractedResult + ") key(" + entry.getKey() + ")", actualResults.contains(extractedResult));
        actualResults.add(extractedResult);
        // perform any custom assert checks on document
        for (final DocumentChecker check : checkers) {
            check.assertValid(document);
        }
    }
    log.info("total records found(" + actualResults.size() + ") expected(" + expected.size() + ")");
    // ensure that the complete expected result set exists
    if (expected.size() > actualResults.size()) {
        final Set<String> notFound = new HashSet<>(expected);
        notFound.removeAll(actualResults);
        for (final String m : notFound) {
            log.error("missing result(" + m + ")");
        }
    } else if (expected.size() < actualResults.size()) {
        final Set<String> extra = new HashSet<>(actualResults);
        extra.removeAll(expected);
        for (final String r : extra) {
            log.error("unexpected result(" + r + ")");
        }
    }
    Assert.assertEquals("results do not match expected", expected.size(), actualResults.size());
    Assert.assertTrue("expected and actual values do not match", expected.containsAll(actualResults));
    Assert.assertTrue("expected and actual values do not match", actualResults.containsAll(expected));
}

Example 3 with TypeAttribute

use of datawave.query.attributes.TypeAttribute in project datawave by NationalSecurityAgency.

the class EventDataQueryExpressionVisitor method extractTypes.

public static Set<Type> extractTypes(AttributeFactory attrFactory, String fieldName, String fieldValue, Key key) {
    final Set<Type> types = new HashSet<>();
    final Queue<Attribute<?>> attrQueue = new LinkedList<>();
    attrQueue.add(attrFactory.create(fieldName, fieldValue, key, true));
    Attribute<?> attr;
    while ((attr = attrQueue.poll()) != null) {
        if (TypeAttribute.class.isAssignableFrom(attr.getClass())) {
            TypeAttribute dta = (TypeAttribute) attr;
            Type t = dta.getType();
            types.add(t);
        } else if (AttributeBag.class.isAssignableFrom(attr.getClass())) {
            attrQueue.addAll(((AttributeBag<?>) attr).getAttributes());
        } else {
            log.warn("Unexpected attribute type when extracting type: " + attr.getClass().getCanonicalName());
        }
    }
    return types;
}

Example 4 with TypeAttribute

use of datawave.query.attributes.TypeAttribute in project datawave by NationalSecurityAgency.

the class CompositeFunctionsTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms, Connector connector, ShardQueryLogic logic) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    logic.setMaxEvaluationPipelines(1);
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID");
        if (attr == null) {
            attr = d.get("UUID.0");
        }
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
}

Example 5 with TypeAttribute

use of datawave.query.attributes.TypeAttribute in project datawave by NationalSecurityAgency.

the class IvaratorInterruptTest method runTestQuery.

protected void runTestQuery(List<String> expected, String querystr, Date startDate, Date endDate, Map<String, String> extraParms) throws Exception {
    log.debug("runTestQuery");
    log.trace("Creating QueryImpl");
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    logic.setMaxEvaluationPipelines(1);
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    HashSet<String> expectedSet = new HashSet<>(expected);
    HashSet<String> resultSet;
    resultSet = new HashSet<>();
    Set<Document> docs = new HashSet<>();
    for (Map.Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.debug(entry.getKey() + " => " + d);
        Attribute<?> attr = d.get("UUID");
        if (attr == null)
            attr = d.get("UUID.0");
        Assert.assertNotNull("Result Document did not contain a 'UUID'", attr);
        Assert.assertTrue("Expected result to be an instance of DatwawaveTypeAttribute, was: " + attr.getClass().getName(), attr instanceof TypeAttribute || attr instanceof PreNormalizedAttribute);
        TypeAttribute<?> UUIDAttr = (TypeAttribute<?>) attr;
        String UUID = UUIDAttr.getType().getDelegate().toString();
        Assert.assertTrue("Received unexpected UUID: " + UUID, expected.contains(UUID));
        resultSet.add(UUID);
        docs.add(d);
    }
    if (expected.size() > resultSet.size()) {
        expectedSet.addAll(expected);
        expectedSet.removeAll(resultSet);
        for (String s : expectedSet) {
            log.warn("Missing: " + s);
        }
    }
    if (!expected.containsAll(resultSet)) {
        log.error("Expected results " + expected + " differ form actual results " + resultSet);
    }
    Assert.assertTrue("Expected results " + expected + " differ form actual results " + resultSet, expected.containsAll(resultSet));
    Assert.assertEquals("Unexpected number of records", expected.size(), resultSet.size());
}