Examples with Operation io.pravega.segmentstore.server.logs.operations.Operation used on opensource projects

Example 46 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class SegmentAggregator method flushNormally.

/**
 * Repeatedly flushes the contents of the Aggregator to the Storage as long as something immediate needs to be flushed,
 * such as a Seal or Merge operation.
 *
 * @param timer Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the result from the flush operation.
 */
private CompletableFuture<FlushResult> flushNormally(TimeoutTimer timer) {
    assert this.state.get() == AggregatorState.Writing : "flushNormally cannot be called if state == " + this.state;
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flushNormally", this.operations.size());
    FlushResult result = new FlushResult();
    AtomicBoolean canContinue = new AtomicBoolean(true);
    return Futures.loop(canContinue::get, () -> flushOnce(timer), partialResult -> {
        canContinue.set(partialResult.getFlushedBytes() + partialResult.getMergedBytes() > 0);
        result.withFlushResult(partialResult);
    }, this.executor).thenApply(v -> {
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flushNormally", traceId, result);
        return result;
    });
}

Example 47 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class SegmentAggregator method initialize.

// endregion
// region Operations
/**
 * Initializes the SegmentAggregator by pulling information from the given Storage.
 *
 * @param timeout  Timeout for the operation.
 * @return A CompletableFuture that, when completed, will indicate that the operation finished successfully. If any
 * errors occurred during the operation, the Future will be completed with the appropriate exception.
 */
CompletableFuture<Void> initialize(Duration timeout) {
    Exceptions.checkNotClosed(isClosed(), this);
    Preconditions.checkState(this.state.get() == AggregatorState.NotInitialized, "SegmentAggregator has already been initialized.");
    assert this.handle.get() == null : "non-null handle but state == " + this.state.get();
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "initialize");
    return openWrite(this.metadata.getName(), this.handle, timeout).thenAcceptAsync(segmentInfo -> {
        // Check & Update StorageLength in metadata.
        if (this.metadata.getStorageLength() != segmentInfo.getLength()) {
            if (this.metadata.getStorageLength() >= 0) {
                // Only log warning if the StorageLength has actually been initialized, but is different.
                log.warn("{}: SegmentMetadata has a StorageLength ({}) that is different than the actual one ({}) - updating metadata.", this.traceObjectId, this.metadata.getStorageLength(), segmentInfo.getLength());
            }
            // It is very important to keep this value up-to-date and correct.
            this.metadata.setStorageLength(segmentInfo.getLength());
        }
        // Check if the Storage segment is sealed, but it's not in metadata (this is 100% indicative of some data corruption happening).
        if (segmentInfo.isSealed()) {
            if (!this.metadata.isSealed()) {
                throw new CompletionException(new DataCorruptionException(String.format("Segment '%s' is sealed in Storage but not in the metadata.", this.metadata.getName())));
            }
            if (!this.metadata.isSealedInStorage()) {
                this.metadata.markSealedInStorage();
                log.warn("{}: Segment is sealed in Storage but metadata does not reflect that - updating metadata.", this.traceObjectId);
            }
        }
        log.info("{}: Initialized. StorageLength = {}, Sealed = {}.", this.traceObjectId, segmentInfo.getLength(), segmentInfo.isSealed());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "initialize", traceId);
        setState(AggregatorState.Writing);
    }, this.executor).exceptionally(ex -> {
        ex = Exceptions.unwrap(ex);
        if (ex instanceof StreamSegmentNotExistsException) {
            // Segment does not exist anymore. This is a real possibility during recovery, in the following cases:
            // * We already processed a Segment Deletion but did not have a chance to checkpoint metadata
            // * We processed a TransactionMergeOperation but did not have a chance to ack/truncate the DataSource
            // Update metadata, just in case it is not already updated.
            this.metadata.markDeleted();
            log.warn("{}: Segment does not exist in Storage. Ignoring all further operations on it.", this.traceObjectId, ex);
            setState(AggregatorState.Writing);
            LoggerHelpers.traceLeave(log, this.traceObjectId, "initialize", traceId);
        } else {
            // Other kind of error - re-throw.
            throw new CompletionException(ex);
        }
        return null;
    });
}

Example 48 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class SegmentAggregator method flushPendingAppends.

/**
 * Flushes all Append Operations that can be flushed up to the maximum allowed flush size.
 *
 * @param timeout  Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain the result from the flush operation.
 */
private CompletableFuture<FlushResult> flushPendingAppends(Duration timeout) {
    // Gather an InputStream made up of all the operations we can flush.
    FlushArgs flushArgs;
    try {
        flushArgs = getFlushArgs();
    } catch (DataCorruptionException ex) {
        return Futures.failedFuture(ex);
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flushPendingAppends");
    if (flushArgs.getLength() == 0) {
        // Nothing to flush.
        FlushResult result = new FlushResult();
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flushPendingAppends", traceId, result);
        return CompletableFuture.completedFuture(result);
    }
    // Flush them.
    InputStream inputStream = flushArgs.getStream();
    return this.storage.write(this.handle.get(), this.metadata.getStorageLength(), inputStream, flushArgs.getLength(), timeout).thenApplyAsync(v -> {
        FlushResult result = updateStatePostFlush(flushArgs);
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flushPendingAppends", traceId, result);
        return result;
    }, this.executor).exceptionally(ex -> {
        if (Exceptions.unwrap(ex) instanceof BadOffsetException) {
            // We attempted to write at an offset that already contained other data. This can happen for a number of
            // reasons, but we do not have enough information here to determine why. We need to enter reconciliation
            // mode, and hope for the best.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}

Example 49 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class SegmentAggregator method mergeWith.

/**
 * Merges the Transaction StreamSegment with given metadata into this one at the current offset.
 *
 * @param transactionMetadata The metadata of the Transaction StreamSegment to merge.
 * @param timer               Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the number of bytes that were merged into this
 * StreamSegment. If failed, the Future will contain the exception that caused it.
 */
private CompletableFuture<FlushResult> mergeWith(UpdateableSegmentMetadata transactionMetadata, MergeTransactionOperation mergeOp, TimeoutTimer timer) {
    if (transactionMetadata.isDeleted()) {
        return Futures.failedFuture(new DataCorruptionException(String.format("Attempted to merge with deleted Transaction segment '%s'.", transactionMetadata.getName())));
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "mergeWith", transactionMetadata.getId(), transactionMetadata.getName(), transactionMetadata.isSealedInStorage());
    FlushResult result = new FlushResult();
    if (!transactionMetadata.isSealedInStorage() || transactionMetadata.getLength() > transactionMetadata.getStorageLength()) {
        // Nothing to do. Given Transaction is not eligible for merger yet.
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return CompletableFuture.completedFuture(result);
    }
    AtomicLong mergedLength = new AtomicLong();
    return this.storage.getStreamSegmentInfo(transactionMetadata.getName(), timer.getRemaining()).thenAcceptAsync(transProperties -> {
        // Check that the Storage agrees with our metadata (if not, we have a problem ...)
        if (transProperties.getLength() != transactionMetadata.getStorageLength()) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' cannot be merged into parent '%s' because its metadata disagrees with the Storage. Metadata.StorageLength=%d, Storage.StorageLength=%d", transactionMetadata.getName(), this.metadata.getName(), transactionMetadata.getStorageLength(), transProperties.getLength())));
        }
        if (transProperties.getLength() != mergeOp.getLength()) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' cannot be merged into parent '%s' because the declared length in the operation disagrees with the Storage. Operation.Length=%d, Storage.StorageLength=%d", transactionMetadata.getName(), this.metadata.getName(), mergeOp.getLength(), transProperties.getLength())));
        }
        mergedLength.set(transProperties.getLength());
    }, this.executor).thenComposeAsync(v1 -> storage.concat(this.handle.get(), mergeOp.getStreamSegmentOffset(), transactionMetadata.getName(), timer.getRemaining()), this.executor).thenComposeAsync(v2 -> storage.getStreamSegmentInfo(this.metadata.getName(), timer.getRemaining()), this.executor).thenApplyAsync(segmentProperties -> {
        // We have processed a MergeTransactionOperation, pop the first operation off and decrement the counter.
        StorageOperation processedOperation = this.operations.removeFirst();
        assert processedOperation != null && processedOperation instanceof MergeTransactionOperation : "First outstanding operation was not a MergeTransactionOperation";
        assert ((MergeTransactionOperation) processedOperation).getTransactionSegmentId() == transactionMetadata.getId() : "First outstanding operation was a MergeTransactionOperation for the wrong Transaction id.";
        int newCount = this.mergeTransactionCount.decrementAndGet();
        assert newCount >= 0 : "Negative value for mergeTransactionCount";
        // Post-merger validation. Verify we are still in agreement with the storage.
        long expectedNewLength = this.metadata.getStorageLength() + mergedLength.get();
        if (segmentProperties.getLength() != expectedNewLength) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' was merged into parent '%s' but the parent segment has an unexpected StorageLength after the merger. Previous=%d, MergeLength=%d, Expected=%d, Actual=%d", transactionMetadata.getName(), this.metadata.getName(), segmentProperties.getLength(), mergedLength.get(), expectedNewLength, segmentProperties.getLength())));
        }
        updateMetadata(segmentProperties);
        updateMetadataForTransactionPostMerger(transactionMetadata);
        this.lastFlush.set(this.timer.getElapsed());
        result.withMergedBytes(mergedLength.get());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return result;
    }, this.executor).exceptionally(ex -> {
        Throwable realEx = Exceptions.unwrap(ex);
        if (realEx instanceof BadOffsetException || realEx instanceof StreamSegmentNotExistsException) {
            // We either attempted to write at an offset that already contained other data or the Transaction
            // Segment no longer exists. This can happen for a number of reasons, but we do not have enough
            // information here to determine why. We need to enter reconciliation mode, and hope for the best.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}

Example 50 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class SegmentAggregator method flush.

// endregion
// region Flushing and Merging
/**
 * Flushes the contents of the Aggregator to the Storage.
 *
 * @param timeout Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain a summary of the flush operation. If any errors
 * occurred during the flush, the Future will be completed with the appropriate exception.
 */
CompletableFuture<FlushResult> flush(Duration timeout) {
    ensureInitializedAndNotClosed();
    if (this.metadata.isDeleted()) {
        // Segment has been deleted; don't do anything else.
        return CompletableFuture.completedFuture(new FlushResult());
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "flush");
    TimeoutTimer timer = new TimeoutTimer(timeout);
    CompletableFuture<FlushResult> result;
    try {
        switch(this.state.get()) {
            case Writing:
                result = flushNormally(timer);
                break;
            case ReconciliationNeeded:
                result = beginReconciliation(timer).thenComposeAsync(v -> reconcile(timer), this.executor);
                break;
            case Reconciling:
                result = reconcile(timer);
                break;
            // $CASES-OMITTED$
            default:
                result = Futures.failedFuture(new IllegalStateException(String.format("Unexpected state for SegmentAggregator (%s) for segment '%s'.", this.state, this.metadata.getName())));
                break;
        }
    } catch (Exception ex) {
        // Convert synchronous errors into async errors - it's easier to handle on the receiving end.
        result = Futures.failedFuture(ex);
    }
    return result.thenApply(r -> {
        LoggerHelpers.traceLeave(log, this.traceObjectId, "flush", traceId, r);
        return r;
    });
}