Examples with Metadata datawave.webservice.query.result.event.Metadata used on opensource projects

Example 1 with Metadata

use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTransformer method readFromCache.

@Override
public List<Object> readFromCache(List<CacheableQueryRow> cacheableQueryRowList) {
    List<Object> eventList = new ArrayList<>();
    for (CacheableQueryRow cqr : cacheableQueryRowList) {
        if (this.variableFieldList == null) {
            this.variableFieldList = cqr.getVariableColumnNames();
        }
        Map<String, String> markings = cqr.getMarkings();
        String dataType = cqr.getDataType();
        String internalId = cqr.getEventId();
        String row = cqr.getRow();
        EventBase event = responseObjectFactory.getEvent();
        event.setMarkings(markings);
        Metadata metadata = new Metadata();
        metadata.setDataType(dataType);
        metadata.setInternalId(internalId);
        metadata.setRow(row);
        metadata.setTable(logic.getTableName());
        event.setMetadata(metadata);
        List<FieldBase> fieldList = new ArrayList<>();
        Map<String, String> columnValueMap = cqr.getColumnValues();
        for (Map.Entry<String, String> entry : columnValueMap.entrySet()) {
            String columnName = entry.getKey();
            String columnValue = entry.getValue();
            String columnVisibility = cqr.getColumnVisibility(columnName);
            Long columnTimestamp = cqr.getColumnTimestamp(columnName);
            Map<String, String> columnMarkings = cqr.getColumnMarkings(columnName);
            FieldBase field = responseObjectFactory.getField();
            field.setName(columnName);
            field.setMarkings(columnMarkings);
            field.setColumnVisibility(columnVisibility);
            field.setTimestamp(columnTimestamp);
            field.setValue(columnValue);
            fieldList.add(field);
        }
        event.setFields(fieldList);
        eventList.add(event);
    }
    return eventList;
}

Example 2 with Metadata

use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTransformer method writeToCache.

@Override
public List<CacheableQueryRow> writeToCache(Object o) throws QueryException {
    List<CacheableQueryRow> cqoList = new ArrayList<>();
    EventBase event = (EventBase) o;
    CacheableQueryRowImpl cqo = new CacheableQueryRowImpl();
    Metadata metadata = event.getMetadata();
    cqo.setColFam(metadata.getDataType() + ":" + cqo.getEventId());
    cqo.setDataType(metadata.getDataType());
    cqo.setEventId(metadata.getInternalId());
    cqo.setRow(metadata.getRow());
    List<FieldBase> fields = event.getFields();
    for (FieldBase f : fields) {
        cqo.addColumn(f.getName(), f.getTypedValue(), f.getMarkings(), f.getColumnVisibility(), f.getTimestamp());
    }
    // set the size in bytes using the initial event size as an approximation
    cqo.setSizeInBytes(event.getSizeInBytes());
    cqoList.add(cqo);
    return cqoList;
}

Example 3 with Metadata

use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTransformer method transform.

@Override
public EventBase transform(Entry<Key, Value> input) {
    log.debug("Transform got " + input);
    @SuppressWarnings("unchecked") Entry<Key, Value> entry = (Entry<Key, Value>) input;
    if (entry.getKey() == null && entry.getValue() == null) {
        return null;
    }
    if (null == entry.getKey() || null == entry.getValue()) {
        throw new IllegalArgumentException("Null key or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
    }
    EventBase event = responseObjectFactory.getEvent();
    ColumnVisibility columnVisibility = new ColumnVisibility(entry.getKey().getColumnVisibility());
    Map<String, String> markings;
    try {
        markings = this.markingFunctions.translateFromColumnVisibilityForAuths(columnVisibility, this.auths);
    } catch (Exception e1) {
        throw new RuntimeException("could not make markings from: " + columnVisibility);
    }
    event.setMarkings(markings);
    List<FieldBase> fields = new ArrayList<>();
    Key key = entry.getKey();
    String row = key.getRow().toString();
    String cf = key.getColumnFamily().toString();
    String cq = key.getColumnQualifier().toString();
    String cv = key.getColumnVisibility().toString();
    fields.add(makeField("VALUE", markings, cv, 0L, row));
    /**
     * Added query model to alias FIELD
     */
    fields.add(makeField("FIELD", markings, cv, 0L, myQueryModel.aliasFieldNameReverseModel(cf)));
    fields.add(makeField("DATE", markings, cv, 0L, cq.substring(0, 8)));
    fields.add(makeField("DATA TYPE", markings, cv, 0L, cq.substring(9)));
    // Parse the UID.List object from the value
    Uid.List uidList = null;
    long count = 0;
    try {
        uidList = Uid.List.parseFrom(entry.getValue().get());
        if (null != uidList) {
            count = uidList.getCOUNT();
        }
    } catch (InvalidProtocolBufferException e) {
        log.error("Failed to parse Uid List", e);
    }
    fields.add(makeField("RECORD COUNT", markings, cv, 0L, Long.toString(count)));
    event.setFields(fields);
    Metadata metadata = new Metadata();
    String id = logic.getTableName() + ":" + row + ":" + cf + ":" + cq;
    metadata.setInternalId(UUID.nameUUIDFromBytes(id.getBytes()).toString());
    metadata.setDataType(entry.getKey().getColumnFamily().toString());
    metadata.setRow(entry.getKey().getRow().toString());
    metadata.setTable(logic.getTableName());
    event.setMetadata(metadata);
    return event;
}

Example 4 with Metadata

use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.

the class TermFrequencyQueryTransformer method transform.

@Override
public DefaultEvent transform(Entry<Key, Value> entry) throws EmptyObjectException {
    if (entry.getKey() == null && entry.getValue() == null) {
        return null;
    }
    if (entry.getKey() == null || entry.getValue() == null) {
        throw new IllegalArgumentException("Null keyy or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
    }
    TermFrequencyKeyValue tfkv;
    try {
        tfkv = TermFrequencyKeyValueFactory.parse(entry.getKey(), entry.getValue(), auths, markingFunctions);
    } catch (Exception e) {
        throw new IllegalArgumentException("Unable to parse visibility", e);
    }
    DefaultEvent e = new DefaultEvent();
    e.setMarkings(tfkv.getMarkings());
    Metadata m = new Metadata();
    m.setRow(tfkv.getShardId());
    m.setDataType(tfkv.getShardId());
    m.setInternalId(tfkv.getUid());
    e.setMetadata(m);
    List<DefaultField> fields = ImmutableList.of(createField(tfkv, entry, "FIELD_NAME", tfkv.getFieldName()), createField(tfkv, entry, "FIELD_VALUE", tfkv.getFieldValue()), createField(tfkv, entry, "OFFSET_COUNT", String.valueOf(tfkv.getCount())), createField(tfkv, entry, "OFFSETS", tfkv.getOffsets().toString()));
    e.setFields(fields);
    return e;
}

Example 5 with Metadata

use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.

the class FieldIndexCountQueryTransformer method transform.

@Override
public EventBase transform(Entry<String, Tuple> entry) {
    if (entry.getKey() == null && entry.getValue() == null) {
        return null;
    }
    if (null == entry.getKey() || null == entry.getValue()) {
        throw new IllegalArgumentException("Null key or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
    }
    String key = entry.getKey();
    Tuple val = entry.getValue();
    Map<String, String> markings = null;
    try {
        markings = this.markingFunctions.translateFromColumnVisibilityForAuths(val.getColumnVisibility(), this.auths);
    } catch (Exception e) {
        log.error("could not translate " + val.getColumnVisibility() + " to markings, skipping entry");
        return null;
    }
    if (null == markings || markings.isEmpty()) {
        // can't process this one because we did not have valid security markings
        log.error("Transformer visibility interpreter was null, skipping entry");
        return null;
    }
    EventBase event = this.responseObjectFactory.getEvent();
    event.setMarkings(markings);
    List<FieldBase> fields = new ArrayList<>();
    int idx1 = key.indexOf(Constants.NULL_BYTE_STRING);
    if (idx1 == -1) {
        log.error("Invalid entry to FieldIndexCountQueryTransformer");
    }
    // NOTE: fieldName has already been re-mapped by query model in the queryLogic class.
    String fieldName = key.substring(0, idx1);
    String fieldValue;
    String dataType = null;
    // parse from the end to handle nasty field values
    int idx2 = key.lastIndexOf(Constants.NULL_BYTE_STRING);
    if (idx2 > idx1) {
        fieldValue = key.substring(idx1 + 1, idx2);
        dataType = key.substring(idx2 + 1);
    } else {
        fieldValue = key.substring(idx1 + 1);
    }
    String colVis = new String(val.getColumnVisibility().getExpression());
    fields.add(this.makeField("FIELD", markings, colVis, 0L, fieldName));
    fields.add(this.makeField("VALUE", markings, colVis, 0L, fieldValue));
    if (dataType != null) {
        fields.add(this.makeField("DATATYPE", markings, colVis, 0L, dataType));
    }
    fields.add(this.makeField("MOST_RECENT", markings, colVis, 0L, Long.toString(val.getMaxTimestamp())));
    fields.add(this.makeField("RECORD_COUNT", markings, colVis, 0L, Long.toString(val.getCount())));
    event.setFields(fields);
    Text uid = new Text(fieldName);
    TextUtil.textAppend(uid, fieldValue);
    if (null != dataType) {
        TextUtil.textAppend(uid, dataType);
    }
    Metadata metadata = new Metadata();
    metadata.setDataType(Constants.EMPTY_STRING);
    metadata.setInternalId((UID.builder().newId(uid.getBytes())).toString());
    metadata.setRow(Constants.EMPTY_STRING);
    metadata.setTable(logic.getTableName());
    event.setMetadata(metadata);
    if (log.isTraceEnabled()) {
        log.trace("Transformer returning: ");
        log.trace("\tfieldName: " + fieldName);
        log.trace("\tfieldValue: " + fieldValue);
        log.trace("\tdataType: " + ((null != dataType) ? dataType : ""));
        log.trace("\tcount: " + Long.toString(val.getCount()));
        log.trace("\ttimestamp: " + Long.toString(val.getMaxTimestamp()));
    }
    return event;
}