Examples with Attributes datawave.query.attributes.Attributes used on opensource projects

Example 1 with Attributes

use of datawave.query.attributes.Attributes in project datawave by NationalSecurityAgency.

the class JexlEvaluation method apply.

@Override
public boolean apply(Tuple3<Key, Document, DatawaveJexlContext> input) {
    Object o = script.execute(input.third());
    if (log.isTraceEnabled()) {
        log.trace("Evaluation of " + query + " against " + input.third() + " returned " + o);
    }
    boolean matched = isMatched(o);
    // Add delayed info to document
    if (matched && input.third() instanceof DelayedNonEventIndexContext) {
        ((DelayedNonEventIndexContext) input.third()).populateDocument(input.second());
    }
    if (arithmetic instanceof HitListArithmetic) {
        HitListArithmetic hitListArithmetic = (HitListArithmetic) arithmetic;
        if (matched) {
            Document document = input.second();
            Attributes attributes = new Attributes(input.second().isToKeep());
            for (ValueTuple hitTuple : hitListArithmetic.getHitTuples()) {
                ColumnVisibility cv = null;
                String term = hitTuple.getFieldName() + ':' + hitTuple.getValue();
                if (hitTuple.getSource() != null) {
                    cv = hitTuple.getSource().getColumnVisibility();
                }
                // fall back to extracting column visibility from document
                if (cv == null) {
                    // get the visibility for the record with this hit
                    cv = HitListArithmetic.getColumnVisibilityForHit(document, term);
                // if no visibility computed, then there were no hits that match fields still in the document......
                }
                if (cv != null) {
                    // unused
                    // will force an update to make the metadata valid
                    long timestamp = document.getTimestamp();
                    Content content = new Content(term, document.getMetadata(), document.isToKeep());
                    content.setColumnVisibility(cv);
                    attributes.add(content);
                }
            }
            if (attributes.size() > 0) {
                document.put(HIT_TERM_FIELD, attributes);
            }
        }
        hitListArithmetic.clear();
    }
    return matched;
}

Example 2 with Attributes

use of datawave.query.attributes.Attributes in project datawave by NationalSecurityAgency.

the class LimitFields method apply.

@Override
public Entry<Key, Document> apply(Entry<Key, Document> entry) {
    // key is the limited field name with _ORIGINAL_COUNT appended,
    // value will be set to the original count of that field in the document
    Map<String, Integer> limitedFieldCounts = new HashMap<>();
    Document document = entry.getValue();
    Map<String, String> hitTermMap = this.getHitTermMap(document);
    Multimap<String, Attribute<? extends Comparable<?>>> reducedMap = LinkedListMultimap.create();
    Map<String, Integer> countForFieldMap = Maps.newHashMap();
    // maps from the key with NO grouping context to a multimap of
    // key WITH grouping context to attributes:
    // DIRECTION : [DIRECTION.1 : [over,under], DIRECTION.2 : [sideways,down]]
    LoadingCache<String, Multimap<String, Attribute<? extends Comparable<?>>>> hits = CacheBuilder.newBuilder().build(new CacheLoader<String, Multimap<String, Attribute<? extends Comparable<?>>>>() {

        public Multimap<String, Attribute<? extends Comparable<?>>> load(String key) {
            return LinkedListMultimap.create();
        }
    });
    // maps from the key with NO grouping context to a multimap of
    // key WITH grouping context to attributes:
    // DIRECTION : [DIRECTION.1 : [over,under], DIRECTION.2 : [sideways,down]]
    @SuppressWarnings("serial") LoadingCache<String, Multimap<String, Attribute<? extends Comparable<?>>>> misses = CacheBuilder.newBuilder().build(new CacheLoader<String, Multimap<String, Attribute<? extends Comparable<?>>>>() {

        public Multimap<String, Attribute<? extends Comparable<?>>> load(String key) {
            return LinkedListMultimap.create();
        }
    });
    for (Map.Entry<String, Attribute<? extends Comparable<?>>> de : document.entrySet()) {
        String keyWithGrouping = de.getKey();
        String keyNoGrouping = keyWithGrouping;
        // if we have grouping context on, remove the grouping context
        if (keyNoGrouping.indexOf('.') != -1) {
            keyNoGrouping = keyNoGrouping.substring(0, keyNoGrouping.indexOf('.'));
        }
        // limit value for _ANYFIELD_
        if (this.limitFieldsMap.containsKey("_ANYFIELD_") && this.limitFieldsMap.containsKey(keyNoGrouping) == false) {
            this.limitFieldsMap.put(keyNoGrouping, this.limitFieldsMap.get("_ANYFIELD_"));
            log.trace("added " + keyNoGrouping + " - " + this.limitFieldsMap.get(keyNoGrouping) + " to the limitFieldsMap because of the _ANYFIELD_ entry");
        }
        if (this.limitFieldsMap.containsKey(keyNoGrouping)) {
            // look for the key without the grouping context
            if (log.isTraceEnabled())
                log.trace("limitFieldsMap contains " + keyNoGrouping);
            Attribute<?> attr = de.getValue();
            // used below if you un-comment to get all hits
            int limit = this.limitFieldsMap.get(keyNoGrouping);
            if (attr instanceof Attributes) {
                Attributes attrs = (Attributes) attr;
                Set<Attribute<? extends Comparable<?>>> attrSet = attrs.getAttributes();
                for (Attribute<? extends Comparable<?>> value : attrSet) {
                    manageHitsAndMisses(keyWithGrouping, keyNoGrouping, value, hitTermMap, hits, misses, countForFieldMap);
                }
            } else {
                manageHitsAndMisses(keyWithGrouping, keyNoGrouping, attr, hitTermMap, hits, misses, countForFieldMap);
            }
        }
    }
    for (String keyNoGrouping : countForFieldMap.keySet()) {
        int limit = this.limitFieldsMap.get(keyNoGrouping);
        Multimap<String, Attribute<? extends Comparable<?>>> hitMap = hits.getUnchecked(keyNoGrouping);
        for (String keyWithGrouping : hitMap.keySet()) {
            for (Attribute<? extends Comparable<?>> value : hitMap.get(keyWithGrouping)) {
                // if(limit <= 0) break; // comment this line if you want to get ALL hits even if the limit is exceeded
                reducedMap.put(keyWithGrouping, value);
                limit--;
            }
        }
        Multimap<String, Attribute<? extends Comparable<?>>> missMap = misses.getUnchecked(keyNoGrouping);
        for (String keyWithGrouping : missMap.keySet()) {
            for (Attribute<? extends Comparable<?>> value : missMap.get(keyWithGrouping)) {
                if (limit <= 0)
                    break;
                reducedMap.put(keyWithGrouping, value);
                limit--;
            }
        }
        if (log.isTraceEnabled()) {
            log.trace("reducedMap:" + reducedMap);
            log.trace("mapOfHits:" + hits.asMap());
            log.trace("mapOfMisses:" + misses.asMap());
        }
        // only generate an original count if a field was reduced
        if (countForFieldMap.get(keyNoGrouping) > this.limitFieldsMap.get(keyNoGrouping)) {
            limitedFieldCounts.put(keyNoGrouping + ORIGINAL_COUNT_SUFFIX, countForFieldMap.get(keyNoGrouping));
        }
    }
    // mutate the document with the changes collected in the above loop
    applyCounts(document, limitedFieldCounts);
    Map<String, Multimap<String, Attribute<? extends Comparable<?>>>> toRemove = Maps.newLinkedHashMap();
    toRemove.putAll(hits.asMap());
    toRemove.putAll(misses.asMap());
    makeReduction(document, toRemove, reducedMap);
    return entry;
}

Example 3 with Attributes

use of datawave.query.attributes.Attributes in project datawave by NationalSecurityAgency.

the class QueryLogicTestHarness method assertLogicResults.

// =============================================
// assert methods
/**
 * Determines if the correct results were obtained for a query.
 *
 * @param logic
 *            key/value response data
 * @param expected
 *            list of key values expected within response data
 * @param checkers
 *            list of additional validation methods
 */
public void assertLogicResults(BaseQueryLogic<Map.Entry<Key, Value>> logic, Collection<String> expected, List<DocumentChecker> checkers) {
    Set<String> actualResults = new HashSet<>();
    if (log.isDebugEnabled()) {
        log.debug("    ======  expected id(s)  ======");
        for (String e : expected) {
            log.debug("id(" + e + ")");
        }
    }
    for (Map.Entry<Key, Value> entry : logic) {
        if (FinalDocumentTrackingIterator.isFinalDocumentKey(entry.getKey())) {
            continue;
        }
        final Document document = this.deserializer.apply(entry).getValue();
        // check all of the types to ensure that all are keepers as defined in the
        // AttributeFactory class
        int count = 0;
        for (Attribute<? extends Comparable<?>> attribute : document.getAttributes()) {
            if (attribute instanceof TimingMetadata) {
            // ignore
            } else if (attribute instanceof Attributes) {
                Attributes attrs = (Attributes) attribute;
                Collection<Class<?>> types = new HashSet<>();
                for (Attribute<? extends Comparable<?>> attr : attrs.getAttributes()) {
                    count++;
                    if (attr instanceof TypeAttribute) {
                        Type<? extends Comparable<?>> type = ((TypeAttribute<?>) attr).getType();
                        if (Objects.nonNull(type)) {
                            types.add(type.getClass());
                        }
                    }
                }
                Assert.assertEquals(AttributeFactory.getKeepers(types), types);
            } else {
                count++;
            }
        }
        // ignore empty documents (possible when only passing FinalDocument back)
        if (count == 0) {
            continue;
        }
        // parse the document
        String extractedResult = this.parser.parse(entry.getKey(), document);
        log.debug("result(" + extractedResult + ") key(" + entry.getKey() + ") document(" + document + ")");
        // verify expected results
        Assert.assertNotNull("extracted result", extractedResult);
        Assert.assertFalse("duplicate result(" + extractedResult + ") key(" + entry.getKey() + ")", actualResults.contains(extractedResult));
        actualResults.add(extractedResult);
        // perform any custom assert checks on document
        for (final DocumentChecker check : checkers) {
            check.assertValid(document);
        }
    }
    log.info("total records found(" + actualResults.size() + ") expected(" + expected.size() + ")");
    // ensure that the complete expected result set exists
    if (expected.size() > actualResults.size()) {
        final Set<String> notFound = new HashSet<>(expected);
        notFound.removeAll(actualResults);
        for (final String m : notFound) {
            log.error("missing result(" + m + ")");
        }
    } else if (expected.size() < actualResults.size()) {
        final Set<String> extra = new HashSet<>(actualResults);
        extra.removeAll(expected);
        for (final String r : extra) {
            log.error("unexpected result(" + r + ")");
        }
    }
    Assert.assertEquals("results do not match expected", expected.size(), actualResults.size());
    Assert.assertTrue("expected and actual values do not match", expected.containsAll(actualResults));
    Assert.assertTrue("expected and actual values do not match", actualResults.containsAll(expected));
}

Example 4 with Attributes

use of datawave.query.attributes.Attributes in project datawave by NationalSecurityAgency.

the class ResponseFieldChecker method assertValid.

/**
 * Verifies the query response document contains all of the return fields.
 *
 * @param doc
 *            query response document
 */
@Override
public void assertValid(final Document doc) {
    for (final String field : this.fields) {
        final Attribute val = doc.get(field);
        Assert.assertNotNull("missing return field(" + field + ")", val);
        if (val instanceof Attributes) {
            Attributes multiAttr = (Attributes) val;
            for (Attribute attr : multiAttr.getAttributes()) {
                Assert.assertNotNull("missing metadata for field(" + field + ")", attr.getMetadata());
            }
        } else {
            Assert.assertNotNull("missing metadata for field(" + field + ")", val.getMetadata());
        }
    }
    for (final String field : this.missing) {
        final Attribute val = doc.get(field);
        Assert.assertNull("blacklisted return field(" + field + ")", val);
    }
}

Example 5 with Attributes

use of datawave.query.attributes.Attributes in project datawave by NationalSecurityAgency.

the class HitsAreAlwaysIncludedCommonalityTokenTest method runTestQuery.

protected void runTestQuery(Connector connector, String queryString, Date startDate, Date endDate, Map<String, String> extraParms, Collection<String> goodResults) throws Exception {
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(queryString);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    Set<Document> docs = new HashSet<>();
    for (Entry<Key, Value> entry : logic) {
        Document d = deserializer.apply(entry).getValue();
        log.trace(entry.getKey() + " => " + d);
        docs.add(d);
        Attribute hitAttribute = d.get(JexlEvaluation.HIT_TERM_FIELD);
        if (hitAttribute instanceof Attributes) {
            Attributes attributes = (Attributes) hitAttribute;
            for (Attribute attr : attributes.getAttributes()) {
                if (attr instanceof Content) {
                    Content content = (Content) attr;
                    Assert.assertTrue(goodResults.contains(content.getContent()));
                }
            }
        } else if (hitAttribute instanceof Content) {
            Content content = (Content) hitAttribute;
            Assert.assertTrue(goodResults.contains(content.getContent()));
        }
        // remove from goodResults as we find the expected return fields
        log.debug("goodResults: " + goodResults);
        Map<String, Attribute<? extends Comparable<?>>> dictionary = d.getDictionary();
        log.debug("dictionary:" + dictionary);
        for (Entry<String, Attribute<? extends Comparable<?>>> dictionaryEntry : dictionary.entrySet()) {
            Attribute<? extends Comparable<?>> attribute = dictionaryEntry.getValue();
            if (attribute instanceof Attributes) {
                for (Attribute attr : ((Attributes) attribute).getAttributes()) {
                    String toFind = dictionaryEntry.getKey() + ":" + attr;
                    boolean found = goodResults.remove(toFind);
                    if (found)
                        log.debug("removed " + toFind);
                    else
                        log.debug("Did not remove " + toFind);
                }
            } else {
                String toFind = dictionaryEntry.getKey() + ":" + dictionaryEntry.getValue();
                boolean found = goodResults.remove(toFind);
                if (found)
                    log.debug("removed " + toFind);
                else
                    log.debug("Did not remove " + toFind);
            }
        }
        Assert.assertTrue(goodResults + " was not empty", goodResults.isEmpty());
    }
    Assert.assertTrue("No docs were returned!", !docs.isEmpty());
}