use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.
the class EventQueryTransformer method transform.
@Override
public EventBase transform(Entry<Key, Value> entry) {
Key key = entry.getKey();
Value val = entry.getValue();
if (entry.getKey() == null && entry.getValue() == null)
return null;
if (null == entry.getKey() || null == entry.getValue()) {
throw new IllegalArgumentException("Null key or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
}
EventBase event = this.responseObjectFactory.getEvent();
Map<String, String> markings = null;
try {
markings = this.markingFunctions.translateFromColumnVisibilityForAuths(new ColumnVisibility(key.getColumnVisibility()), this.auths);
} catch (Exception e) {
log.error("could not translate " + key.getColumnVisibility() + " to markings, skipping entry");
return null;
}
if (null == markings || markings.isEmpty()) {
// can't process this one because we did not have valid security markings
log.error("Transformer visibility interpreter was null, skipping entry");
return null;
}
ByteArrayInputStream bais = new ByteArrayInputStream(entry.getValue().get());
Input i = new Input(bais);
eventFields = kryo.readObject(i, EventFields.class);
i.close();
String row = entry.getKey().getRow().toString();
String colf = entry.getKey().getColumnFamily().toString();
String colq = entry.getKey().getColumnQualifier().toString();
// evaluated by using the returnUidMapper (@see datawave.core.iterators.EvaluatingIterator: aggregateAltEvent)
if (!colq.equals("")) {
colf = colq;
}
int sepIndex = colf.indexOf(Constants.NULL_BYTE_STRING);
String baseUid = colf.substring(sepIndex + 1);
Set<FieldBase<?>> values = new HashSet<>();
String origFieldName = null;
String fieldName = null;
// Hold unique Column Visibilities and merge them at the end
// for the overall event ColumnVisibility.
Set<ColumnVisibility> visibilitiesToMerge = new HashSet<>();
for (Entry<String, Collection<FieldValue>> e : eventFields.asMap().entrySet()) {
origFieldName = e.getKey();
if (this.qm != null) {
fieldName = this.qm.aliasFieldNameReverseModel(origFieldName);
} else {
fieldName = origFieldName;
}
for (FieldValue fv : e.getValue()) {
visibilitiesToMerge.add(fv.getVisibility());
try {
Map<String, String> fieldMarkings = this.markingFunctions.translateFromColumnVisibility(fv.getVisibility());
String value = new String(fv.getValue(), Charset.forName("UTF-8"));
// if this is a content field name, then replace the value with the uid
if (getContentFieldNames().contains(fieldName)) {
value = baseUid;
}
values.add(this.makeField(fieldName, fieldMarkings, new String(fv.getVisibility().getExpression()), entry.getKey().getTimestamp(), value));
} catch (Exception e1) {
throw new RuntimeException("could not make markings from: " + fv.getVisibility());
}
}
}
ColumnVisibility columnVisibility = null;
try {
columnVisibility = this.markingFunctions.combine(visibilitiesToMerge);
event.setMarkings(this.markingFunctions.translateFromColumnVisibility(columnVisibility));
} catch (Exception e1) {
throw new RuntimeException("could not make markings from: " + columnVisibility);
}
event.setFields(new ArrayList<>(values));
Metadata metadata = new Metadata();
String[] colfParts = StringUtils.split(colf, '\0');
if (colfParts.length >= 1) {
metadata.setDataType(colfParts[0]);
}
if (colfParts.length >= 2) {
metadata.setInternalId(colfParts[1]);
}
if (this.tableName != null) {
metadata.setTable(this.tableName);
}
metadata.setRow(row);
event.setMetadata(metadata);
if (eventQueryDataDecoratorTransformer != null) {
event = (EventBase<?, ?>) eventQueryDataDecoratorTransformer.transform(event);
}
// assign an estimate of the event size
// in practice this is about 6 times the size of the kryo bytes
event.setSizeInBytes(entry.getValue().getSize() * 6);
return event;
}
use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.
the class EventQueryTransformerSupport method readFromCache.
@Override
public List<Object> readFromCache(List<CacheableQueryRow> cacheableQueryRowList) {
List<Object> eventList = new ArrayList<>();
for (CacheableQueryRow cqr : cacheableQueryRowList) {
Map<String, String> markings = cqr.getMarkings();
String dataType = cqr.getDataType();
String internalId = cqr.getEventId();
String row = cqr.getRow();
EventBase event = this.responseObjectFactory.getEvent();
event.setMarkings(markings);
Metadata metadata = new Metadata();
metadata.setDataType(dataType);
metadata.setInternalId(internalId);
metadata.setRow(row);
metadata.setTable(logic.getTableName());
event.setMetadata(metadata);
List<FieldBase<?>> fieldList = new ArrayList<>();
Map<String, String> columnValueMap = cqr.getColumnValues();
for (Entry<String, String> entry : columnValueMap.entrySet()) {
String columnName = entry.getKey();
String columnValue = entry.getValue();
Map<String, String> columnMarkings = cqr.getColumnMarkings(columnName);
String columnVisibility = cqr.getColumnVisibility(columnName);
Long columnTimestamp = cqr.getColumnTimestamp(columnName);
FieldBase<?> field = this.makeField(columnName, columnMarkings, columnVisibility, columnTimestamp, columnValue);
fieldList.add(field);
}
event.setFields(fieldList);
eventList.add(event);
}
return eventList;
}
use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.
the class EventQueryTransformerSupport method writeToCache.
@Override
public List<CacheableQueryRow> writeToCache(Object o) throws QueryException {
List<CacheableQueryRow> cqoList = new ArrayList<>();
EventBase<?, ?> event = (EventBase<?, ?>) o;
CacheableQueryRow cqo = this.responseObjectFactory.getCacheableQueryRow();
Metadata metadata = event.getMetadata();
cqo.setColFam(metadata.getDataType() + ":" + cqo.getEventId());
cqo.setDataType(metadata.getDataType());
cqo.setEventId(metadata.getInternalId());
cqo.setRow(metadata.getRow());
List<? extends FieldBase<?>> fields = event.getFields();
for (FieldBase<?> f : fields) {
cqo.addColumn(f.getName(), f.getTypedValue(), f.getMarkings(), f.getColumnVisibility(), f.getTimestamp());
}
cqoList.add(cqo);
return cqoList;
}
use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.
the class ContentQueryTransformer method transform.
@Override
public DefaultEvent transform(Entry<Key, Value> entry) {
if (entry.getKey() == null && entry.getValue() == null)
return null;
if (null == entry.getKey() || null == entry.getValue()) {
throw new IllegalArgumentException("Null key or value. Key:" + entry.getKey() + ", Value: " + entry.getValue());
}
ContentKeyValue ckv;
try {
ckv = ContentKeyValueFactory.parse(entry.getKey(), entry.getValue(), auths, markingFunctions);
} catch (Exception e1) {
throw new IllegalArgumentException("Unable to parse visibility", e1);
}
DefaultEvent e = new DefaultEvent();
DefaultField field = new DefaultField();
e.setMarkings(ckv.getMarkings());
Metadata m = new Metadata();
m.setRow(ckv.getShardId());
m.setDataType(ckv.getDatatype());
m.setInternalId(ckv.getUid());
e.setMetadata(m);
field.setMarkings(ckv.getMarkings());
field.setName(ckv.getViewName());
field.setTimestamp(entry.getKey().getTimestamp());
field.setValue(ckv.getContents());
List<DefaultField> fields = new ArrayList<>();
fields.add(field);
e.setFields(fields);
return e;
}
use of datawave.webservice.query.result.event.Metadata in project datawave by NationalSecurityAgency.
the class LookupUUIDUtil method createContentQueryStrings.
/*
* Create contentQuery strings based on the specified events, if any. A contentQuery string from one event would look like the following example: <p>
* "DOCUMENT:shardId/datatype/uid" <p> <b>Note:</b> An attempt was made to concatenate multiple events into a single OR'd expression, but the
* ContentQueryTable only supports one term at a time.
*
* @param eventResponse the response from the UUID query
*
* @return a list of batched content query strings
*/
private List<StringBuilder> createContentQueryStrings(final EventQueryResponseBase eventResponse) {
// Initialize a flag that can be flipped to support multi-term ContentQueryTable expressions
// at some unknown point in the future
boolean multiTermExpressionsSupported = true;
// Declare the returned list of query strings
final List<StringBuilder> contentQueryStrings;
// Build query strings based on an optimized all-event response
if (eventResponse instanceof AllEventMockResponse) {
final AllEventMockResponse allEventResponse = (AllEventMockResponse) eventResponse;
final AbstractUUIDLookupCriteria validatedCriteria = allEventResponse.getLookupCriteria();
contentQueryStrings = this.createContentQueryStrings(validatedCriteria, multiTermExpressionsSupported);
} else // Otherwise, handle "normally" built query strings
{
// Create the list
contentQueryStrings = new LinkedList<>();
// Get the entire list of queried events
final List<EventBase> unbatchedEvents = eventResponse.getEvents();
// Iterate through the unbatched events and fill the batched set of query strings
StringBuilder contentQuery = null;
int eventCounter = 0;
for (final EventBase<?, ?> event : unbatchedEvents) {
// Get the event's shard table info
final Metadata metadata = event.getMetadata();
final String row = metadata.getRow();
final String dataType = metadata.getDataType();
final String internalId = metadata.getInternalId();
// Increment the counter
eventCounter++;
// Conditionally initialize a new query string and the event counter
if ((null == contentQuery) || (!multiTermExpressionsSupported) || ((this.maxAllowedBatchLookupUUIDs > 0) && (eventCounter > this.maxAllowedBatchLookupUUIDs))) {
contentQuery = new StringBuilder();
contentQueryStrings.add(contentQuery);
eventCounter = 1;
} else // Conditionally append an OR operator
if (contentQuery.length() > 0) {
contentQuery.append(SPACE);
}
// Append the content query criteria
contentQuery.append(DOCUMENT_FIELD_NAME).append(row);
contentQuery.append(FORWARD_SLASH).append(dataType);
contentQuery.append(FORWARD_SLASH).append(internalId);
}
}
return contentQueryStrings;
}
Aggregations