Examples with Cardinality datawave.query.attributes.Cardinality used on opensource projects

Example 1 with Cardinality

use of datawave.query.attributes.Cardinality in project datawave by NationalSecurityAgency.

the class FacetedTransformer method buildFacets.

/**
 * Accepts an attribute. The document data will be placed into the value of the Field.
 *
 * @param documentKey
 * @param fieldName
 * @param attr
 * @param topLevelColumnVisibility
 * @param markingFunctions
 * @return
 */
protected Collection<FieldCardinalityBase> buildFacets(Key documentKey, String fieldName, Attribute<?> attr, ColumnVisibility topLevelColumnVisibility, MarkingFunctions markingFunctions) {
    Set<FieldCardinalityBase> myFields = new HashSet<>();
    if (attr instanceof Attributes) {
        Attributes attributeList = Attributes.class.cast(attr);
        for (Attribute<?> embeddedAttr : attributeList.getAttributes()) myFields.addAll(buildFacets(documentKey, fieldName, embeddedAttr, topLevelColumnVisibility, markingFunctions));
    } else {
        // Use the markings on the Field if we're returning the markings to the client
        if (attr instanceof Cardinality) {
            try {
                Cardinality card = (Cardinality) attr;
                FieldValueCardinality v = card.getContent();
                StringType value = new StringType();
                value.setDelegate(v.toString());
                value.setNormalizedValue(v.toString());
                FieldCardinalityBase fc = this.responseObjectFactory.getFieldCardinality();
                fc.setField(v.getFieldName());
                // reduces colvis based on
                fc.setMarkings(markingFunctions.translateFromColumnVisibilityForAuths(attr.getColumnVisibility(), auths));
                // visibility
                fc.setColumnVisibility(new String(markingFunctions.translateToColumnVisibility(fc.getMarkings()).flatten()));
                fc.setLower(v.getFloorValue());
                fc.setUpper(v.getCeilingValue());
                fc.setCardinality(v.getEstimate().cardinality());
                myFields.add(fc);
            } catch (Exception e) {
                log.error("unable to process markings:" + e);
            }
        }
    }
    return myFields;
}

Example 2 with Cardinality

use of datawave.query.attributes.Cardinality in project datawave by NationalSecurityAgency.

the class DocumentCountCardinality method apply.

/*
     * (non-Javadoc)
     * 
     * @see com.google.common.base.Function#apply(java.lang.Object)
     */
@Override
public Entry<Key, Document> apply(Entry<Key, Document> input) {
    Document prevDoc = input.getValue();
    Key key = input.getKey();
    // reduce the key to the document key pieces only
    key = new Key(key.getRow(), key.getColumnFamily());
    Document newDoc = new Document();
    if (prevDoc.size() > 0) {
        String dayOrShard = key.getRow().toString();
        if (FacetedSearchType.DAY_COUNT == type) {
            dayOrShard = dayOrShard.substring(0, dayOrShard.indexOf("_"));
        }
        FieldValueCardinality fvC = new FieldValueCardinality();
        fvC.setContent(dayOrShard);
        if (setDocIds)
            fvC.setDocId(dayOrShard);
        // use the overall visibility and timestamp
        Cardinality card = new Cardinality(fvC, key, true);
        newDoc.put(dayOrShard, card);
    }
    return Maps.immutableEntry(key, newDoc);
}

Example 3 with Cardinality

use of datawave.query.attributes.Cardinality in project datawave by NationalSecurityAgency.

the class AttributeToCardinality method apply.

/*
     * (non-Javadoc)
     * 
     * @see com.google.common.base.Function#apply(java.lang.Object)
     */
@Override
public Entry<Key, Document> apply(Entry<Key, Document> input) {
    Document prevDoc = input.getValue();
    Key key = input.getKey();
    // for cardinalities, only use the visibility metadata
    Key metadata = new Key(EMPTY_TEXT, EMPTY_TEXT, EMPTY_TEXT, prevDoc.getColumnVisibility(), -1);
    Document newDoc = new Document();
    Map<?, ?> dictionary = (Map<?, ?>) prevDoc.getData();
    TreeMap<String, Attribute<? extends Comparable<?>>> newDictionary = Maps.newTreeMap();
    DatawaveKey parser = new DatawaveKey(input.getKey());
    for (Entry<?, ?> attrE : dictionary.entrySet()) {
        Entry<String, Attribute<?>> attr = (Entry<String, Attribute<?>>) attrE;
        if (!attr.getKey().equals(Document.DOCKEY_FIELD_NAME)) {
            Attribute<?> attribute = attr.getValue();
            if (attribute instanceof Attributes) {
                Attributes attrs = (Attributes) attribute;
                Attributes newAttrs = new Attributes(attrs.isToKeep());
                for (Attribute<?> attributeItem : attrs.getAttributes()) {
                    Cardinality card = null;
                    if (attributeItem instanceof Cardinality) {
                        card = (Cardinality) attributeItem;
                    } else {
                        FieldValueCardinality fvC = new FieldValueCardinality();
                        fvC.setContent(attributeItem.getData().toString());
                        fvC.setDoc(prevDoc);
                        card = new Cardinality(fvC, metadata, attrs.isToKeep());
                        if (log.isTraceEnabled())
                            log.trace("Adding from attributes " + attr.getKey() + " " + attributeItem.getData());
                    }
                    newAttrs.add(card);
                }
                newDictionary.put(attr.getKey(), newAttrs);
            } else {
                Cardinality card = null;
                if (attribute instanceof Cardinality) {
                    card = (Cardinality) attribute;
                } else {
                    FieldValueCardinality fvC = new FieldValueCardinality();
                    fvC.setContent(attribute.getData().toString());
                    fvC.setDoc(prevDoc);
                    card = new Cardinality(fvC, metadata, attribute.isToKeep());
                    if (log.isTraceEnabled())
                        log.trace("Adding " + parser.getUid() + " " + attr.getKey() + " " + attribute.getData() + " " + fvC.getEstimate().cardinality());
                }
                newDictionary.put(attr.getKey(), card);
            }
        }
    }
    newDoc.putAll(newDictionary.entrySet().iterator(), false);
    return Maps.immutableEntry(key, newDoc);
}

Example 4 with Cardinality

use of datawave.query.attributes.Cardinality in project datawave by NationalSecurityAgency.

the class CardinalityAggregator method apply.

@Override
public Key apply(SortedKeyValueIterator<Key, Value> itr, Document doc, AttributeFactory attrs) throws IOException {
    Key key = itr.getTopKey();
    Text row = key.getRow();
    ByteSequence pointer = parsePointer(key.getColumnQualifierData());
    Key nextKey = key;
    while (nextKey != null && samePointer(row, pointer, nextKey)) {
        DatawaveKey topKey = new DatawaveKey(nextKey);
        String field = topKey.getFieldName();
        String value = topKey.getFieldValue();
        FieldValueCardinality fvC = null;
        byte[] currentValue = itr.getTopValue().get();
        try {
            if (currentValue.length > 0) {
                fvC = new FieldValueCardinality(HyperLogLogPlus.Builder.build(currentValue));
                if (log.isTraceEnabled()) {
                    log.trace("Set cardinality from FI value");
                }
            }
        } catch (Exception e) {
            if (log.isTraceEnabled()) {
                log.trace("Exception encountered " + e);
            }
        }
        if (null == fvC) {
            if (log.isTraceEnabled())
                log.trace("Building cardinality for " + topKey.getUid());
            fvC = new FieldValueCardinality();
            if (setDocIds)
                fvC.setDocId(topKey.getUid());
        }
        fvC.setContent(value);
        // for cardinalities, only use the visibility metadata
        Key metadata = new Key(EMPTY_TEXT, EMPTY_TEXT, EMPTY_TEXT, itr.getTopKey().getColumnVisibility(), -1);
        Cardinality card = new Cardinality(fvC, metadata, doc.isToKeep());
        // only keep fields that are index only
        card.setToKeep(fieldsToKeep == null || fieldsToKeep.contains(JexlASTHelper.removeGroupingContext(field)));
        doc.put(field, card);
        key = nextKey;
        itr.next();
        nextKey = (itr.hasTop() ? itr.getTopKey() : null);
    }
    return TLD.buildParentKey(row, pointer, TLD.parseFieldAndValueFromFI(key.getColumnFamilyData(), key.getColumnQualifierData()), key.getColumnVisibility(), key.getTimestamp());
}

Example 5 with Cardinality

use of datawave.query.attributes.Cardinality in project datawave by NationalSecurityAgency.

the class FacetTableFunction method apply.

@Override
public Entry<Key, Document> apply(Entry<Key, Value> input) {
    Key key = input.getKey();
    Document newDoc = new Document();
    try {
        String[] fields = StringUtils.split(key.getColumnFamily().toString(), "\u0000");
        String[] fieldValues = StringUtils.split(key.getRow().toString(), "\u0000");
        FieldValueCardinality fvc = new FieldValueCardinality(HyperLogLogPlus.Builder.build(input.getValue().get()));
        fvc.setFieldName(fields[1]);
        fvc.setContent(fieldValues[1]);
        Cardinality card = new Cardinality(fvc, key, false);
        newDoc.put(fields[1], card);
        return Maps.immutableEntry(key, newDoc);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}