Search in sources :

Example 21 with FieldBase

use of datawave.webservice.query.result.event.FieldBase in project datawave by NationalSecurityAgency.

the class ShardQueryCountTableTransformer method writeToCache.

@Override
public List<CacheableQueryRow> writeToCache(Object o) throws QueryException {
    List<CacheableQueryRow> cqoList = new ArrayList<>();
    EventBase event = (EventBase) o;
    CacheableQueryRowImpl cqo = new CacheableQueryRowImpl();
    Metadata metadata = event.getMetadata();
    cqo.setColFam(metadata.getDataType() + ":" + cqo.getEventId());
    cqo.setDataType(metadata.getDataType());
    cqo.setEventId(metadata.getInternalId());
    cqo.setRow(metadata.getRow());
    List<FieldBase> fields = event.getFields();
    for (FieldBase f : fields) {
        cqo.addColumn(f.getName(), f.getTypedValue(), f.getMarkings(), f.getColumnVisibility(), f.getTimestamp());
    }
    // set the size in bytes using the initial event size as an approximation
    cqo.setSizeInBytes(event.getSizeInBytes());
    cqoList.add(cqo);
    return cqoList;
}
Also used : EventBase(datawave.webservice.query.result.event.EventBase) ArrayList(java.util.ArrayList) Metadata(datawave.webservice.query.result.event.Metadata) FieldBase(datawave.webservice.query.result.event.FieldBase) CacheableQueryRow(datawave.webservice.query.cachedresults.CacheableQueryRow) CacheableQueryRowImpl(datawave.webservice.query.cachedresults.CacheableQueryRowImpl)

Example 22 with FieldBase

use of datawave.webservice.query.result.event.FieldBase in project datawave by NationalSecurityAgency.

the class ShardQueryCountTableTransformer method readFromCache.

@Override
public List<Object> readFromCache(List<CacheableQueryRow> cacheableQueryRowList) {
    List<Object> eventList = new ArrayList<>();
    for (CacheableQueryRow cqr : cacheableQueryRowList) {
        if (this.variableFieldList == null) {
            this.variableFieldList = cqr.getVariableColumnNames();
        }
        Map<String, String> markings = cqr.getMarkings();
        String dataType = cqr.getDataType();
        String internalId = cqr.getEventId();
        String row = cqr.getRow();
        EventBase event = responseObjectFactory.getEvent();
        event.setMarkings(markings);
        Metadata metadata = new Metadata();
        metadata.setDataType(dataType);
        metadata.setInternalId(internalId);
        metadata.setRow(row);
        event.setMetadata(metadata);
        List<FieldBase> fieldList = new ArrayList<>();
        Map<String, String> columnValueMap = cqr.getColumnValues();
        for (Map.Entry<String, String> entry : columnValueMap.entrySet()) {
            String columnName = entry.getKey();
            String columnValue = entry.getValue();
            String columnVisibility = cqr.getColumnVisibility(columnName);
            Long columnTimestamp = cqr.getColumnTimestamp(columnName);
            Map<String, String> columnMarkings = cqr.getColumnMarkings(columnName);
            FieldBase field = responseObjectFactory.getField();
            field.setName(columnName);
            field.setMarkings(columnMarkings);
            field.setColumnVisibility(columnVisibility);
            field.setTimestamp(columnTimestamp);
            field.setValue(columnValue);
            fieldList.add(field);
        }
        event.setFields(fieldList);
        eventList.add(event);
    }
    return eventList;
}
Also used : EventBase(datawave.webservice.query.result.event.EventBase) ArrayList(java.util.ArrayList) Metadata(datawave.webservice.query.result.event.Metadata) CacheableQueryRow(datawave.webservice.query.cachedresults.CacheableQueryRow) FieldBase(datawave.webservice.query.result.event.FieldBase) Map(java.util.Map)

Example 23 with FieldBase

use of datawave.webservice.query.result.event.FieldBase in project datawave by NationalSecurityAgency.

the class ShardQueryCountTableTransformer method transform.

@Override
public EventBase transform(Entry<Long, ColumnVisibility> untypedEntry) {
    Long count = untypedEntry.getKey();
    ColumnVisibility vis = untypedEntry.getValue();
    Map<String, String> markings;
    try {
        markings = markingFunctions.translateFromColumnVisibilityForAuths(vis, auths);
    } catch (Exception e1) {
        throw new IllegalArgumentException("Unable to translate markings", e1);
    }
    EventBase e = this.responseObjectFactory.getEvent();
    e.setMarkings(markings);
    FieldBase field = this.makeField(COUNT_CELL, markings, vis, System.currentTimeMillis(), count);
    e.setMarkings(markings);
    List<FieldBase> fields = new ArrayList<>();
    fields.add(field);
    e.setFields(fields);
    Metadata metadata = new Metadata();
    metadata.setDataType(Constants.EMPTY_STRING);
    // There is only one item returned for the entire query logic.
    metadata.setInternalId(field.getName());
    metadata.setRow(Constants.EMPTY_STRING);
    e.setMetadata(metadata);
    return e;
}
Also used : EventBase(datawave.webservice.query.result.event.EventBase) FieldBase(datawave.webservice.query.result.event.FieldBase) ArrayList(java.util.ArrayList) Metadata(datawave.webservice.query.result.event.Metadata) ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility) QueryException(datawave.webservice.query.exception.QueryException) Exception(datawave.marking.MarkingFunctions.Exception)

Example 24 with FieldBase

use of datawave.webservice.query.result.event.FieldBase in project datawave by NationalSecurityAgency.

the class EventQueryTransformer method transform.

@Override
public EventBase transform(Entry<Key, Value> entry) {
    Key key = entry.getKey();
    Value val = entry.getValue();
    if (entry.getKey() == null && entry.getValue() == null)
        return null;
    if (null == entry.getKey() || null == entry.getValue()) {
        throw new IllegalArgumentException("Null key or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
    }
    EventBase event = this.responseObjectFactory.getEvent();
    Map<String, String> markings = null;
    try {
        markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths);
    } catch (Exception e) {
        log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry");
        return null;
    }
    if (null == markings || markings.isEmpty()) {
        // can't process this one because we did not have valid security markings
        log.error("Transformer visibility interpreter was null, skipping entry");
        return null;
    }
    ByteArrayInputStream bais = new ByteArrayInputStream(entry.getValue().get());
    Input i = new Input(bais);
    eventFields = kryo.readObject(i, EventFields.class);
    i.close();
    String row = entry.getKey().getRow().toString();
    String colf = entry.getKey().getColumnFamily().toString();
    String colq = entry.getKey().getColumnQualifier().toString();
    // evaluated by using the returnUidMapper (@see datawave.core.iterators.EvaluatingIterator: aggregateAltEvent)
    if (!colq.equals("")) {
        colf = colq;
    }
    int sepIndex = colf.indexOf(Constants.NULL_BYTE_STRING);
    String baseUid = colf.substring(sepIndex + 1);
    Set<FieldBase<?>> values = new HashSet<>();
    String origFieldName = null;
    String fieldName = null;
    // Hold unique Column Visibilities and merge them at the end
    // for the overall event ColumnVisibility.
    Set<ColumnVisibility> visibilitiesToMerge = new HashSet<>();
    for (Entry<String, Collection<FieldValue>> e : eventFields.asMap().entrySet()) {
        origFieldName = e.getKey();
        if (this.qm != null) {
            fieldName = this.qm.aliasFieldNameReverseModel(origFieldName);
        } else {
            fieldName = origFieldName;
        }
        for (FieldValue fv : e.getValue()) {
            visibilitiesToMerge.add(fv.getVisibility());
            try {
                Map<String, String> fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility());
                String value = new String(fv.getValue(), Charset.forName("UTF-8"));
                // if this is a content field name, then replace the value with the uid
                if (getContentFieldNames().contains(fieldName)) {
                    value = baseUid;
                }
                values.add(this.makeField(fieldName, fieldMarkings, new String(fv.getVisibility().getExpression()), entry.getKey().getTimestamp(), value));
            } catch (Exception e1) {
                throw new RuntimeException("could not make markings from: " + fv.getVisibility());
            }
        }
    }
    ColumnVisibility columnVisibility = null;
    try {
        columnVisibility = this.markingFunctions.combine(visibilitiesToMerge);
        event.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility));
    } catch (Exception e1) {
        throw new RuntimeException("could not make markings from: " + columnVisibility);
    }
    event.setFields(new ArrayList<>(values));
    Metadata metadata = new Metadata();
    String[] colfParts = StringUtils.split(colf, '\0');
    if (colfParts.length >= 1) {
        metadata.setDataType(colfParts[0]);
    }
    if (colfParts.length >= 2) {
        metadata.setInternalId(colfParts[1]);
    }
    if (this.tableName != null) {
        metadata.setTable(this.tableName);
    }
    metadata.setRow(row);
    event.setMetadata(metadata);
    if (eventQueryDataDecoratorTransformer != null) {
        event = (EventBase<?, ?>) eventQueryDataDecoratorTransformer.transform(event);
    }
    // assign an estimate of the event size
    // in practice this is about 6 times the size of the kryo bytes
    event.setSizeInBytes(entry.getValue().getSize() * 6);
    return event;
}
Also used : EventBase(datawave.webservice.query.result.event.EventBase) Metadata(datawave.webservice.query.result.event.Metadata) Exception(datawave.marking.MarkingFunctions.Exception) EventFields(datawave.query.parser.EventFields) Input(com.esotericsoftware.kryo.io.Input) ByteArrayInputStream(java.io.ByteArrayInputStream) FieldValue(datawave.query.parser.EventFields.FieldValue) Value(org.apache.accumulo.core.data.Value) FieldBase(datawave.webservice.query.result.event.FieldBase) Collection(java.util.Collection) ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility) FieldValue(datawave.query.parser.EventFields.FieldValue) Key(org.apache.accumulo.core.data.Key) HashSet(java.util.HashSet)

Example 25 with FieldBase

use of datawave.webservice.query.result.event.FieldBase in project datawave by NationalSecurityAgency.

the class EventQueryTransformerSupport method readFromCache.

@Override
public List<Object> readFromCache(List<CacheableQueryRow> cacheableQueryRowList) {
    List<Object> eventList = new ArrayList<>();
    for (CacheableQueryRow cqr : cacheableQueryRowList) {
        Map<String, String> markings = cqr.getMarkings();
        String dataType = cqr.getDataType();
        String internalId = cqr.getEventId();
        String row = cqr.getRow();
        EventBase event = this.responseObjectFactory.getEvent();
        event.setMarkings(markings);
        Metadata metadata = new Metadata();
        metadata.setDataType(dataType);
        metadata.setInternalId(internalId);
        metadata.setRow(row);
        metadata.setTable(logic.getTableName());
        event.setMetadata(metadata);
        List<FieldBase<?>> fieldList = new ArrayList<>();
        Map<String, String> columnValueMap = cqr.getColumnValues();
        for (Entry<String, String> entry : columnValueMap.entrySet()) {
            String columnName = entry.getKey();
            String columnValue = entry.getValue();
            Map<String, String> columnMarkings = cqr.getColumnMarkings(columnName);
            String columnVisibility = cqr.getColumnVisibility(columnName);
            Long columnTimestamp = cqr.getColumnTimestamp(columnName);
            FieldBase<?> field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue);
            fieldList.add(field);
        }
        event.setFields(fieldList);
        eventList.add(event);
    }
    return eventList;
}
Also used : EventBase(datawave.webservice.query.result.event.EventBase) ArrayList(java.util.ArrayList) Metadata(datawave.webservice.query.result.event.Metadata) CacheableQueryRow(datawave.webservice.query.cachedresults.CacheableQueryRow) FieldBase(datawave.webservice.query.result.event.FieldBase)

Aggregations

FieldBase (datawave.webservice.query.result.event.FieldBase)32 EventBase (datawave.webservice.query.result.event.EventBase)24 ArrayList (java.util.ArrayList)20 Metadata (datawave.webservice.query.result.event.Metadata)15 CacheableQueryRow (datawave.webservice.query.cachedresults.CacheableQueryRow)9 Map (java.util.Map)6 Exception (datawave.marking.MarkingFunctions.Exception)4 QueryException (datawave.webservice.query.exception.QueryException)4 ColumnVisibility (org.apache.accumulo.core.security.ColumnVisibility)4 QueryImpl (datawave.webservice.query.QueryImpl)3 CacheableQueryRowImpl (datawave.webservice.query.cachedresults.CacheableQueryRowImpl)3 EventQueryResponseBase (datawave.webservice.result.EventQueryResponseBase)3 Date (java.util.Date)3 HashMap (java.util.HashMap)3 TransformIterator (org.apache.commons.collections4.iterators.TransformIterator)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 GenericQueryConfiguration (datawave.webservice.query.configuration.GenericQueryConfiguration)2 DatawaveTransformIterator (datawave.webservice.query.iterator.DatawaveTransformIterator)2 BaseQueryResponse (datawave.webservice.result.BaseQueryResponse)2 DefaultEventQueryResponse (datawave.webservice.result.DefaultEventQueryResponse)2