Examples with WriterFlushResult io.pravega.segmentstore.server.WriterFlushResult used on opensource projects

Example 21 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregator method flushExcess.

/**
 * Flushes as many Append Operations as needed as long as the data inside this SegmentAggregator exceeds size/time thresholds.
 * This will stop when either the thresholds are not exceeded anymore or when a non-Append Operation is encountered.
 *
 * @param timer Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the result from the flush operation.
 */
private CompletableFuture<WriterFlushResult> flushExcess(TimeoutTimer timer) {
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flushExcess");
    WriterFlushResult result = new WriterFlushResult();
    return Futures.loop(this::canContinueFlushingExcess, () -> flushPendingAppends(timer.getRemaining()).thenCompose(flushResult -> flushPendingTruncate(flushResult, timer.getRemaining())), result::withFlushResult, this.executor).thenApply(v -> {
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flushExcess", traceId, result);
        return result;
    });
}

Example 22 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregator method flush.

// endregion
// region Flushing and Merging
/**
 * Flushes the contents of the Aggregator to the Storage.
 *
 * @param force   If true, force-flushes everything accumulated in the {@link SegmentAggregator}, regardless of
 *                the value returned by {@link #mustFlush()}.
 * @param timeout Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain a summary of the flush operation. If any errors
 * occurred during the flush, the Future will be completed with the appropriate exception.
 */
@Override
public CompletableFuture<WriterFlushResult> flush(boolean force, Duration timeout) {
    ensureInitializedAndNotClosed();
    if (this.metadata.isDeletedInStorage()) {
        // Segment has been deleted; don't do anything else.
        return CompletableFuture.completedFuture(new WriterFlushResult());
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flush");
    TimeoutTimer timer = new TimeoutTimer(timeout);
    CompletableFuture<WriterFlushResult> result;
    try {
        switch(this.state.get()) {
            case Writing:
                result = flushNormally(force, timer);
                break;
            case ReconciliationNeeded:
                result = beginReconciliation(timer).thenComposeAsync(v -> reconcile(timer), this.executor);
                break;
            case Reconciling:
                result = reconcile(timer);
                break;
            // $CASES-OMITTED$
            default:
                result = Futures.failedFuture(new IllegalStateException(String.format("Unexpected state for SegmentAggregator (%s) for segment '%s'.", this.state, this.metadata.getName())));
                break;
        }
    } catch (Exception ex) {
        // Convert synchronous errors into async errors - it's easier to handle on the receiving end.
        result = Futures.failedFuture(ex);
    }
    return result.thenApply(r -> {
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flush", traceId, r);
        return r;
    });
}

Example 23 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregator method reconcile.

private CompletableFuture<WriterFlushResult> reconcile(TimeoutTimer timer) {
    ReconciliationState rc = this.reconciliationState.get();
    WriterFlushResult result = new WriterFlushResult();
    if (rc == null) {
        setState(AggregatorState.Writing);
        return CompletableFuture.completedFuture(result);
    } else if (this.hasDeletePending.get()) {
        // Do not bother with anything else. If we know we are going to delete this segment, then do no bother doing
        // any other kind of reconciliation work.
        setState(AggregatorState.Writing);
        return deleteSegment(timer);
    }
    SegmentProperties storageInfo = rc.getStorageInfo();
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "reconcile", rc);
    // Process each Operation in sequence, as long as its starting offset is less than ReconciliationState.getStorageInfo().getLength()
    AtomicBoolean exceededStorageLength = new AtomicBoolean(false);
    return Futures.loop(() -> this.operations.size() > 0 && !exceededStorageLength.get(), () -> {
        StorageOperation op = this.operations.getFirst();
        return reconcileOperation(op, storageInfo, timer).thenApply(partialFlushResult -> {
            if (op.getLastStreamSegmentOffset() >= storageInfo.getLength()) {
                // This operation crosses the boundary of StorageLength. It has been reconciled,
                // and as such it is the last operation that we need to inspect.
                exceededStorageLength.set(true);
            }
            log.info("{}: Reconciled {} ({}).", this.traceObjectId, op, partialFlushResult);
            return partialFlushResult;
        });
    }, result::withFlushResult, this.executor).thenApply(v -> {
        updateMetadata(storageInfo);
        this.reconciliationState.set(null);
        setState(AggregatorState.Writing);
        LoggerHelpers.traceLeave(log, this.traceObjectId, "reconcile", traceId, result);
        return result;
    });
}

Example 24 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregator method updateStatePostFlush.

/**
 * Updates the metadata and the internal state after a flush was completed.
 *
 * @param flushData The arguments used for flushing.
 * @return A FlushResult containing statistics about the flush operation.
 */
private WriterFlushResult updateStatePostFlush(BufferView flushData) {
    // Update the metadata Storage Length, if necessary.
    long newLength = this.metadata.getStorageLength();
    int flushLength = flushData == null ? 0 : flushData.getLength();
    if (flushLength > 0) {
        newLength += flushData.getLength();
        this.metadata.setStorageLength(newLength);
    }
    // Remove Append Operations from the outstanding list as long as every single byte they contain have been committed.
    boolean reachedEnd = false;
    while (this.operations.size() > 0 && !reachedEnd) {
        StorageOperation first = this.operations.getFirst();
        long lastOffset = first.getLastStreamSegmentOffset();
        reachedEnd = lastOffset >= newLength;
        if (!isAppendOperation(first)) {
            // We can only remove Append Operations.
            reachedEnd = true;
        } else if (lastOffset <= newLength) {
            // Fully flushed Append Operation.
            this.operations.removeFirst();
        }
    }
    // Update the last flush checkpoint.
    this.lastFlush.set(this.timer.getElapsed());
    return new WriterFlushResult().withFlushedBytes(flushLength);
}

Example 25 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregator method reconcileMergeOperation.

/**
 * Attempts to reconcile the given MergeSegmentOperation.
 *
 * @param op          The Operation to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<WriterFlushResult> reconcileMergeOperation(MergeSegmentOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    // Verify that the transaction segment is still registered in metadata.
    UpdateableSegmentMetadata transactionMeta = this.dataSource.getStreamSegmentMetadata(op.getSourceSegmentId());
    if (transactionMeta == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the source segment is missing from the metadata.", op), this.metadata, storageInfo));
    }
    // Verify that the operation fits fully within this segment (mergers are atomic - they either merge all or nothing).
    if (op.getLastStreamSegmentOffset() > storageInfo.getLength()) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the source segment is not fully merged into the target.", op), this.metadata, storageInfo));
    }
    // Verify that the transaction segment does not exist in Storage anymore.
    return this.storage.exists(transactionMeta.getName(), timer.getRemaining()).thenComposeAsync(exists -> {
        if (exists) {
            return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the transaction segment still exists in Storage.", op), this.metadata, storageInfo));
        }
        // Clear out any attributes.
        return this.dataSource.deleteAllAttributes(transactionMeta, timer.getRemaining());
    }, this.executor).thenApplyAsync(v -> {
        // Reconciliation complete. Pop the first operation off the list and update the metadata for the transaction segment.
        StorageOperation processedOperation = this.operations.removeFirst();
        assert processedOperation != null && processedOperation instanceof MergeSegmentOperation : "First outstanding operation was not a MergeSegmentOperation";
        int newCount = this.mergeTransactionCount.decrementAndGet();
        assert newCount >= 0 : "Negative value for mergeTransactionCount";
        // Since the operation is already reconciled, the StorageLength of this Segment must be at least
        // the last offset of the operation. We are about to invoke ReadIndex.completeMerge(), which requires
        // that this value be set to at least the last offset of the merged Segment, so we need to ensure it's
        // set now. This will also be set at the end of reconciliation, but we cannot wait until then to invoke
        // the callbacks.
        long minStorageLength = processedOperation.getLastStreamSegmentOffset();
        if (this.metadata.getStorageLength() < minStorageLength) {
            this.metadata.setStorageLength(minStorageLength);
        }
        updateMetadataForTransactionPostMerger(transactionMeta, processedOperation.getStreamSegmentId());
        return new WriterFlushResult().withMergedBytes(op.getLength());
    }, this.executor);
}