Examples with TimeoutTimer io.pravega.common.TimeoutTimer used on opensource projects

Example 71 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class SegmentAggregator method reconcileMergeOperation.

/**
 * Attempts to reconcile the given MergeSegmentOperation.
 *
 * @param op          The Operation to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<WriterFlushResult> reconcileMergeOperation(MergeSegmentOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    // Verify that the transaction segment is still registered in metadata.
    UpdateableSegmentMetadata transactionMeta = this.dataSource.getStreamSegmentMetadata(op.getSourceSegmentId());
    if (transactionMeta == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the source segment is missing from the metadata.", op), this.metadata, storageInfo));
    }
    // Verify that the operation fits fully within this segment (mergers are atomic - they either merge all or nothing).
    if (op.getLastStreamSegmentOffset() > storageInfo.getLength()) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the source segment is not fully merged into the target.", op), this.metadata, storageInfo));
    }
    // Verify that the transaction segment does not exist in Storage anymore.
    return this.storage.exists(transactionMeta.getName(), timer.getRemaining()).thenComposeAsync(exists -> {
        if (exists) {
            return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the transaction segment still exists in Storage.", op), this.metadata, storageInfo));
        }
        // Clear out any attributes.
        return this.dataSource.deleteAllAttributes(transactionMeta, timer.getRemaining());
    }, this.executor).thenApplyAsync(v -> {
        // Reconciliation complete. Pop the first operation off the list and update the metadata for the transaction segment.
        StorageOperation processedOperation = this.operations.removeFirst();
        assert processedOperation != null && processedOperation instanceof MergeSegmentOperation : "First outstanding operation was not a MergeSegmentOperation";
        int newCount = this.mergeTransactionCount.decrementAndGet();
        assert newCount >= 0 : "Negative value for mergeTransactionCount";
        // Since the operation is already reconciled, the StorageLength of this Segment must be at least
        // the last offset of the operation. We are about to invoke ReadIndex.completeMerge(), which requires
        // that this value be set to at least the last offset of the merged Segment, so we need to ensure it's
        // set now. This will also be set at the end of reconciliation, but we cannot wait until then to invoke
        // the callbacks.
        long minStorageLength = processedOperation.getLastStreamSegmentOffset();
        if (this.metadata.getStorageLength() < minStorageLength) {
            this.metadata.setStorageLength(minStorageLength);
        }
        updateMetadataForTransactionPostMerger(transactionMeta, processedOperation.getStreamSegmentId());
        return new WriterFlushResult().withMergedBytes(op.getLength());
    }, this.executor);
}

Example 72 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class AttributeAggregator method flush.

/**
 * Flushes the contents of the Aggregator to the Storage.
 *
 * @param force   If true, force-flushes everything accumulated in the {@link AttributeAggregator}, regardless of
 *                the value returned by {@link #mustFlush()}.
 * @param timeout Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain a summary of the flush operation. If any errors
 * occurred during the flush, the Future will be completed with the appropriate exception.
 */
@Override
public CompletableFuture<WriterFlushResult> flush(boolean force, Duration timeout) {
    Exceptions.checkNotClosed(isClosed(), this);
    if (!force && !mustFlush()) {
        return CompletableFuture.completedFuture(new WriterFlushResult());
    }
    TimeoutTimer timer = new TimeoutTimer(timeout);
    CompletableFuture<Void> result = handleAttributeException(persistPendingAttributes(this.state.getAttributes(), this.state.getLastSequenceNumber(), timer));
    if (this.state.hasSeal()) {
        result = result.thenComposeAsync(v -> handleAttributeException(sealAttributes(timer)), this.executor);
    }
    return result.thenApply(v -> {
        if (this.state.size() > 0) {
            log.debug("{}: Flushed. Count={}, SeqNo={}-{}, Forced={}.", this.traceObjectId, this.state.size(), this.state.getFirstSequenceNumber(), this.state.getLastSequenceNumber(), force);
        }
        WriterFlushResult r = new WriterFlushResult();
        r.withFlushedAttributes(this.state.size());
        this.state.acceptChanges();
        this.lastFlush.set(this.timer.getElapsed());
        return r;
    });
}

Example 73 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class SegmentAggregator method reconcileData.

/**
 * Attempts to reconcile the data for the given AggregatedAppendOperation. Since Append Operations can be partially
 * flushed, reconciliation may be for the full operation or for a part of it.
 *
 * @param op          The AggregatedAppendOperation to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation.
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<Integer> reconcileData(AggregatedAppendOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    BufferView appendData = this.dataSource.getAppendData(op.getStreamSegmentId(), op.getStreamSegmentOffset(), (int) op.getLength());
    if (appendData == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Unable to reconcile operation '%s' because no append data is associated with it.", op), this.metadata, storageInfo));
    }
    // Only read as much data as we need.
    long readLength = Math.min(op.getLastStreamSegmentOffset(), storageInfo.getLength()) - op.getStreamSegmentOffset();
    assert readLength > 0 : "Append Operation to be reconciled is beyond the Segment's StorageLength (" + storageInfo.getLength() + "): " + op;
    // Read all data from storage.
    byte[] storageData = new byte[(int) readLength];
    AtomicInteger reconciledBytes = new AtomicInteger();
    return Futures.loop(() -> reconciledBytes.get() < readLength, () -> this.storage.read(this.handle.get(), op.getStreamSegmentOffset() + reconciledBytes.get(), storageData, reconciledBytes.get(), (int) readLength - reconciledBytes.get(), timer.getRemaining()), bytesRead -> {
        assert bytesRead > 0 : String.format("Unable to make any read progress when reconciling operation '%s' after reading %s bytes.", op, reconciledBytes);
        reconciledBytes.addAndGet(bytesRead);
    }, this.executor).thenApplyAsync(v -> {
        // Compare, byte-by-byte, the contents of the append.
        verifySame(appendData, storageData, op, storageInfo);
        return reconciledBytes.get();
    }, this.executor);
}

Example 74 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class SegmentAggregator method flushPendingAppends.

/**
 * Flushes all Append Operations that can be flushed up to the maximum allowed flush size.
 *
 * @param timeout  Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain the result from the flush operation.
 */
private CompletableFuture<WriterFlushResult> flushPendingAppends(Duration timeout) {
    // Gather an InputStream made up of all the operations we can flush.
    BufferView flushData;
    try {
        flushData = getFlushData();
    } catch (DataCorruptionException ex) {
        return Futures.failedFuture(ex);
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flushPendingAppends");
    // Flush them.
    TimeoutTimer timer = new TimeoutTimer(timeout);
    CompletableFuture<Void> flush;
    if (flushData == null || flushData.getLength() == 0) {
        flush = CompletableFuture.completedFuture(null);
    } else {
        flush = createSegmentIfNecessary(() -> this.storage.write(this.handle.get(), this.metadata.getStorageLength(), flushData.getReader(), flushData.getLength(), timer.getRemaining()), timer.getRemaining());
    }
    return flush.thenApplyAsync(v -> {
        WriterFlushResult result = updateStatePostFlush(flushData);
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flushPendingAppends", traceId, result);
        return result;
    }, this.executor).exceptionally(ex -> {
        if (Exceptions.unwrap(ex) instanceof BadOffsetException) {
            // We attempted to write at an offset that already contained other data. This can happen for a number of
            // reasons, but we do not have enough information here to determine why. We need to enter reconciliation
            // mode, which will determine the actual state of the segment in storage and take appropriate actions.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}

Example 75 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class SegmentAggregator method mergeWith.

/**
 * Merges the Transaction StreamSegment with given metadata into this one at the current offset.
 *
 * @param transactionMetadata The metadata of the Transaction StreamSegment to merge.
 * @param timer               Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the number of bytes that were merged into this
 * StreamSegment. If failed, the Future will contain the exception that caused it.
 */
private CompletableFuture<WriterFlushResult> mergeWith(UpdateableSegmentMetadata transactionMetadata, MergeSegmentOperation mergeOp, TimeoutTimer timer) {
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "mergeWith", transactionMetadata.getId(), transactionMetadata.getName(), transactionMetadata.isSealedInStorage());
    boolean emptySourceSegment = transactionMetadata.getLength() == 0;
    if (transactionMetadata.isDeleted() && !emptySourceSegment) {
        // We came across a deleted source segment that had some data. We need to begin a reconciliation to figure out
        // the actual state of the segments in Storage.
        setState(AggregatorState.ReconciliationNeeded);
        return Futures.failedFuture(new StreamSegmentNotExistsException(transactionMetadata.getName()));
    }
    WriterFlushResult result = new WriterFlushResult();
    CompletableFuture<SegmentProperties> merge;
    if (emptySourceSegment) {
        // We came across a deleted source segment which had no data. No point in attempting to do anything, as any
        // operation involving this segment will complain about it not being there.
        log.warn("{}: Not applying '{}' because source segment is missing or empty.", this.traceObjectId, mergeOp);
        merge = CompletableFuture.completedFuture(this.metadata);
    } else if (!transactionMetadata.isSealedInStorage() || transactionMetadata.getLength() > transactionMetadata.getStorageLength()) {
        // Nothing to do. Given Transaction is not eligible for merger yet.
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return CompletableFuture.completedFuture(result);
    } else {
        merge = mergeInStorage(transactionMetadata, mergeOp, timer);
    }
    // need to make AttributeAggregator aware of merging segments.
    return merge.thenAcceptAsync(segmentProperties -> mergeCompleted(segmentProperties, transactionMetadata, mergeOp), this.executor).thenComposeAsync(v -> this.dataSource.deleteAllAttributes(transactionMetadata, timer.getRemaining()), this.executor).thenApply(v -> {
        this.lastFlush.set(this.timer.getElapsed());
        result.withMergedBytes(mergeOp.getLength());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return result;
    }).exceptionally(ex -> {
        Throwable realEx = Exceptions.unwrap(ex);
        if (realEx instanceof BadOffsetException || realEx instanceof StreamSegmentNotExistsException) {
            // We either attempted to write at an offset that already contained other data or the Transaction
            // Segment no longer exists. This can happen for a number of reasons, but we do not have enough
            // information here to determine why. We need to enter reconciliation mode, and hope for the best.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}