Examples with Bytes org.apache.fluo.api.data.Bytes used on opensource projects

Example 36 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class AggregationResultUpdater method updateAggregateResults.

/**
 * Updates the results of an Aggregation node where its child has emitted a new Binding Set.
 *
 * @param tx - The transaction all Fluo queries will use. (not null)
 * @param childBindingSet - The Binding Set that was omitted by the Aggregation Node's child. (not null)
 * @param aggregationMetadata - The metadata of the Aggregation node whose results will be updated. (not null)
 * @throws Exception The update could not be successfully performed.
 */
public void updateAggregateResults(final TransactionBase tx, final VisibilityBindingSet childBindingSet, final AggregationMetadata aggregationMetadata) throws Exception {
    requireNonNull(tx);
    requireNonNull(childBindingSet);
    requireNonNull(aggregationMetadata);
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Child Binding Set:\n" + childBindingSet + "\n");
    // The Row ID for the Aggregation State that needs to be updated is defined by the Group By variables.
    final String aggregationNodeId = aggregationMetadata.getNodeId();
    final VariableOrder groupByVars = aggregationMetadata.getGroupByVariableOrder();
    final Bytes rowId = makeRowKey(aggregationNodeId, groupByVars, childBindingSet);
    // Load the old state from the bytes if one was found; otherwise initialize the state.
    final Optional<Bytes> stateBytes = Optional.ofNullable(tx.get(rowId, FluoQueryColumns.AGGREGATION_BINDING_SET));
    final AggregationState state;
    if (stateBytes.isPresent()) {
        // Deserialize the old state
        final byte[] bytes = stateBytes.get().toArray();
        state = AGG_STATE_SERDE.deserialize(bytes);
    } else {
        // Initialize a new state.
        state = new AggregationState();
        // If we have group by bindings, their values need to be added to the state's binding set.
        final MapBindingSet bindingSet = state.getBindingSet();
        for (final String variable : aggregationMetadata.getGroupByVariableOrder()) {
            bindingSet.addBinding(childBindingSet.getBinding(variable));
        }
    }
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Before Update: " + LogUtils.clean(state.getBindingSet().toString()) + "\n");
    // Update the visibilities of the result binding set based on the child's visibilities.
    final String oldVisibility = state.getVisibility();
    final String updateVisibilities = VisibilitySimplifier.unionAndSimplify(oldVisibility, childBindingSet.getVisibility());
    state.setVisibility(updateVisibilities);
    // Update the Aggregation State with each Aggregation function included within this group.
    for (final AggregationElement aggregation : aggregationMetadata.getAggregations()) {
        final AggregationType type = aggregation.getAggregationType();
        final AggregationFunction function = FUNCTIONS.get(type);
        if (function == null) {
            throw new RuntimeException("Unrecognized aggregation function: " + type);
        }
        function.update(aggregation, state, childBindingSet);
    }
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "After Update:" + LogUtils.clean(state.getBindingSet().toString()) + "\n");
    // Store the updated state. This will write on top of any old state that was present for the Group By values.
    tx.set(rowId, FluoQueryColumns.AGGREGATION_BINDING_SET, Bytes.of(AGG_STATE_SERDE.serialize(state)));
}

Example 37 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class JoinResultUpdater method fillSiblingBatch.

/**
 * Fetches batch to be processed by scanning over the Span specified by the
 * {@link JoinBatchInformation}. The number of results is less than or equal
 * to the batch size specified by the JoinBatchInformation.
 *
 * @param tx - Fluo transaction in which batch operation is performed
 * @param siblingSpan - span of sibling to retrieve elements to join with
 * @param bsSet- set that batch results are added to
 * @return Set - containing results of sibling scan.
 * @throws Exception
 */
private Optional<RowColumn> fillSiblingBatch(final TransactionBase tx, final Span siblingSpan, final Column siblingColumn, final Set<VisibilityBindingSet> bsSet, final int batchSize) throws Exception {
    final RowScanner rs = tx.scanner().over(siblingSpan).fetch(siblingColumn).byRow().build();
    final Iterator<ColumnScanner> colScannerIter = rs.iterator();
    boolean batchLimitMet = false;
    Bytes row = siblingSpan.getStart().getRow();
    while (colScannerIter.hasNext() && !batchLimitMet) {
        final ColumnScanner colScanner = colScannerIter.next();
        row = colScanner.getRow();
        final Iterator<ColumnValue> iter = colScanner.iterator();
        while (iter.hasNext() && !batchLimitMet) {
            bsSet.add(BS_SERDE.deserialize(iter.next().getValue()));
            // check if batch size has been met and set flag if it has been met
            if (bsSet.size() >= batchSize) {
                batchLimitMet = true;
            }
        }
    }
    if (batchLimitMet) {
        return Optional.of(new RowColumn(row, siblingColumn));
    } else {
        return Optional.absent();
    }
}

Example 38 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class PeriodicQueryUpdater method updatePeriodicBinResults.

/**
 * Uses the {@link PeriodicQueryMetadata} to create a collection of binned BindingSets
 * that are added to Fluo.  Each binned BindingSet is the original BindingSet with an additional
 * Binding that contains the periodic bin id of the BindingSet.
 * @param tx - Fluo Transaction
 * @param bs - VisibilityBindingSet that will be binned
 * @param metadata - PeriodicQueryMetadata used to bin BindingSets
 * @throws Exception
 */
public void updatePeriodicBinResults(TransactionBase tx, VisibilityBindingSet bs, PeriodicQueryMetadata metadata) throws Exception {
    Set<Long> binIds = getBinEndTimes(metadata, bs);
    for (Long id : binIds) {
        // create binding set value bytes
        QueryBindingSet binnedBs = new QueryBindingSet(bs);
        binnedBs.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(id));
        VisibilityBindingSet visibilityBindingSet = new VisibilityBindingSet(binnedBs, bs.getVisibility());
        Bytes periodicBsBytes = BS_SERDE.serialize(visibilityBindingSet);
        // create row
        final Bytes resultRow = makeRowKey(metadata.getNodeId(), metadata.getVariableOrder(), visibilityBindingSet);
        Column col = FluoQueryColumns.PERIODIC_QUERY_BINDING_SET;
        tx.set(resultRow, col, periodicBsBytes);
    }
}

Example 39 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class QueryResultUpdater method updateQueryResults.

/**
 * Updates the results of a Query node when one of its children has added a
 * new Binding Set to its results.
 *
 * @param tx - The transaction all Fluo queries will use. (not null)
 * @param childBindingSet - A binding set that the query's child node has emmitted. (not null)
 * @param queryMetadata - The metadata of the Query whose results will be updated. (not null)
 * @throws Exception A problem caused the update to fail.
 */
public void updateQueryResults(final TransactionBase tx, final VisibilityBindingSet childBindingSet, final QueryMetadata queryMetadata) throws Exception {
    checkNotNull(tx);
    checkNotNull(childBindingSet);
    checkNotNull(queryMetadata);
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "Join Node ID: " + queryMetadata.getNodeId() + "\n" + "Child Node ID: " + queryMetadata.getChildNodeId() + "\n" + "Child Binding Set:\n" + childBindingSet + "\n");
    // Create the query's Binding Set from the child node's binding set.
    final VariableOrder queryVarOrder = queryMetadata.getVariableOrder();
    // Create the Row Key for the result. If the child node groups results, then the key must only contain the Group By variables.
    final Bytes resultRow = makeRowKey(queryMetadata.getNodeId(), queryVarOrder, childBindingSet);
    // Create the Binding Set that goes in the Node Value. It does contain visibilities.
    final Bytes nodeValueBytes = BS_SERDE.serialize(childBindingSet);
    log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "New Binding Set: " + childBindingSet + "\n");
    tx.set(resultRow, FluoQueryColumns.QUERY_BINDING_SET, nodeValueBytes);
}

Example 40 with Bytes

use of org.apache.fluo.api.data.Bytes in project incubator-rya by apache.

the class FilterObserver method parseObservation.

@Override
public Observation parseObservation(final TransactionBase tx, final Bytes row) throws Exception {
    requireNonNull(tx);
    requireNonNull(row);
    // Read the Filter metadata.
    final String filterNodeId = BindingSetRow.makeFromShardedRow(Bytes.of(FILTER_PREFIX), row).getNodeId();
    final FilterMetadata filterMetadata = queryDao.readFilterMetadata(tx, filterNodeId);
    // Read the Visibility Binding Set from the value.
    final Bytes valueBytes = tx.get(row, FluoQueryColumns.FILTER_BINDING_SET);
    final VisibilityBindingSet filterBindingSet = BS_SERDE.deserialize(valueBytes);
    // Figure out which node needs to handle the new metadata.
    final String parentNodeId = filterMetadata.getParentNodeId();
    return new Observation(filterNodeId, filterBindingSet, parentNodeId);
}