use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.
the class AggregationResultUpdater method updateAggregateResults.
/**
* Updates the results of an Aggregation node where its child has emitted a new Binding Set.
*
* @param tx - The transaction all Fluo queries will use. (not null)
* @param childBindingSet - The Binding Set that was omitted by the Aggregation Node's child. (not null)
* @param aggregationMetadata - The metadata of the Aggregation node whose results will be updated. (not null)
* @throws Exception The update could not be successfully performed.
*/
public void updateAggregateResults(final TransactionBase tx, final VisibilityBindingSet childBindingSet, final AggregationMetadata aggregationMetadata) throws Exception {
requireNonNull(tx);
requireNonNull(childBindingSet);
requireNonNull(aggregationMetadata);
log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Child Binding Set:\n" + childBindingSet + "\n");
// The Row ID for the Aggregation State that needs to be updated is defined by the Group By variables.
final String aggregationNodeId = aggregationMetadata.getNodeId();
final VariableOrder groupByVars = aggregationMetadata.getGroupByVariableOrder();
final Bytes rowId = makeRowKey(aggregationNodeId, groupByVars, childBindingSet);
// Load the old state from the bytes if one was found; otherwise initialize the state.
final Optional<Bytes> stateBytes = Optional.ofNullable(tx.get(rowId, FluoQueryColumns.AGGREGATION_BINDING_SET));
final AggregationState state;
if (stateBytes.isPresent()) {
// Deserialize the old state
final byte[] bytes = stateBytes.get().toArray();
state = AGG_STATE_SERDE.deserialize(bytes);
} else {
// Initialize a new state.
state = new AggregationState();
// If we have group by bindings, their values need to be added to the state's binding set.
final MapBindingSet bindingSet = state.getBindingSet();
for (final String variable : aggregationMetadata.getGroupByVariableOrder()) {
bindingSet.addBinding(childBindingSet.getBinding(variable));
}
}
log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Before Update: " + LogUtils.clean(state.getBindingSet().toString()) + "\n");
// Update the visibilities of the result binding set based on the child's visibilities.
final String oldVisibility = state.getVisibility();
final String updateVisibilities = VisibilitySimplifier.unionAndSimplify(oldVisibility, childBindingSet.getVisibility());
state.setVisibility(updateVisibilities);
// Update the Aggregation State with each Aggregation function included within this group.
for (final AggregationElement aggregation : aggregationMetadata.getAggregations()) {
final AggregationType type = aggregation.getAggregationType();
final AggregationFunction function = FUNCTIONS.get(type);
if (function == null) {
throw new RuntimeException("Unrecognized aggregation function: " + type);
}
function.update(aggregation, state, childBindingSet);
}
log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "After Update:" + LogUtils.clean(state.getBindingSet().toString()) + "\n");
// Store the updated state. This will write on top of any old state that was present for the Group By values.
tx.set(rowId, FluoQueryColumns.AGGREGATION_BINDING_SET, Bytes.of(AGG_STATE_SERDE.serialize(state)));
}
use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.
the class JoinResultUpdater method fillSiblingBatch.
/**
* Fetches batch to be processed by scanning over the Span specified by the
* {@link JoinBatchInformation}. The number of results is less than or equal
* to the batch size specified by the JoinBatchInformation.
*
* @param tx - Fluo transaction in which batch operation is performed
* @param siblingSpan - span of sibling to retrieve elements to join with
* @param bsSet- set that batch results are added to
* @return Set - containing results of sibling scan.
* @throws Exception
*/
private Optional<RowColumn> fillSiblingBatch(final TransactionBase tx, final Span siblingSpan, final Column siblingColumn, final Set<VisibilityBindingSet> bsSet, final int batchSize) throws Exception {
final RowScanner rs = tx.scanner().over(siblingSpan).fetch(siblingColumn).byRow().build();
final Iterator<ColumnScanner> colScannerIter = rs.iterator();
boolean batchLimitMet = false;
Bytes row = siblingSpan.getStart().getRow();
while (colScannerIter.hasNext() && !batchLimitMet) {
final ColumnScanner colScanner = colScannerIter.next();
row = colScanner.getRow();
final Iterator<ColumnValue> iter = colScanner.iterator();
while (iter.hasNext() && !batchLimitMet) {
bsSet.add(BS_SERDE.deserialize(iter.next().getValue()));
// check if batch size has been met and set flag if it has been met
if (bsSet.size() >= batchSize) {
batchLimitMet = true;
}
}
}
if (batchLimitMet) {
return Optional.of(new RowColumn(row, siblingColumn));
} else {
return Optional.absent();
}
}
use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.
the class PeriodicQueryUpdater method updatePeriodicBinResults.
/**
* Uses the {@link PeriodicQueryMetadata} to create a collection of binned BindingSets
* that are added to Fluo. Each binned BindingSet is the original BindingSet with an additional
* Binding that contains the periodic bin id of the BindingSet.
* @param tx - Fluo Transaction
* @param bs - VisibilityBindingSet that will be binned
* @param metadata - PeriodicQueryMetadata used to bin BindingSets
* @throws Exception
*/
public void updatePeriodicBinResults(TransactionBase tx, VisibilityBindingSet bs, PeriodicQueryMetadata metadata) throws Exception {
Set<Long> binIds = getBinEndTimes(metadata, bs);
for (Long id : binIds) {
// create binding set value bytes
QueryBindingSet binnedBs = new QueryBindingSet(bs);
binnedBs.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(id));
VisibilityBindingSet visibilityBindingSet = new VisibilityBindingSet(binnedBs, bs.getVisibility());
Bytes periodicBsBytes = BS_SERDE.serialize(visibilityBindingSet);
// create row
final Bytes resultRow = makeRowKey(metadata.getNodeId(), metadata.getVariableOrder(), visibilityBindingSet);
Column col = FluoQueryColumns.PERIODIC_QUERY_BINDING_SET;
tx.set(resultRow, col, periodicBsBytes);
}
}
use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.
the class QueryResultUpdater method updateQueryResults.
/**
* Updates the results of a Query node when one of its children has added a
* new Binding Set to its results.
*
* @param tx - The transaction all Fluo queries will use. (not null)
* @param childBindingSet - A binding set that the query's child node has emmitted. (not null)
* @param queryMetadata - The metadata of the Query whose results will be updated. (not null)
* @throws Exception A problem caused the update to fail.
*/
public void updateQueryResults(final TransactionBase tx, final VisibilityBindingSet childBindingSet, final QueryMetadata queryMetadata) throws Exception {
checkNotNull(tx);
checkNotNull(childBindingSet);
checkNotNull(queryMetadata);
log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Join Node ID: " + queryMetadata.getNodeId() + "\n" + "Child Node ID: " + queryMetadata.getChildNodeId() + "\n" + "Child Binding Set:\n" + childBindingSet + "\n");
// Create the query's Binding Set from the child node's binding set.
final VariableOrder queryVarOrder = queryMetadata.getVariableOrder();
// Create the Row Key for the result. If the child node groups results, then the key must only contain the Group By variables.
final Bytes resultRow = makeRowKey(queryMetadata.getNodeId(), queryVarOrder, childBindingSet);
// Create the Binding Set that goes in the Node Value. It does contain visibilities.
final Bytes nodeValueBytes = BS_SERDE.serialize(childBindingSet);
log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "New Binding Set: " + childBindingSet + "\n");
tx.set(resultRow, FluoQueryColumns.QUERY_BINDING_SET, nodeValueBytes);
}
use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.
the class FilterObserver method parseObservation.
@Override
public Observation parseObservation(final TransactionBase tx, final Bytes row) throws Exception {
requireNonNull(tx);
requireNonNull(row);
// Read the Filter metadata.
final String filterNodeId = BindingSetRow.makeFromShardedRow(Bytes.of(FILTER_PREFIX), row).getNodeId();
final FilterMetadata filterMetadata = queryDao.readFilterMetadata(tx, filterNodeId);
// Read the Visibility Binding Set from the value.
final Bytes valueBytes = tx.get(row, FluoQueryColumns.FILTER_BINDING_SET);
final VisibilityBindingSet filterBindingSet = BS_SERDE.deserialize(valueBytes);
// Figure out which node needs to handle the new metadata.
final String parentNodeId = filterMetadata.getParentNodeId();
return new Observation(filterNodeId, filterBindingSet, parentNodeId);
}
Aggregations