Examples with SegmentProperties io.pravega.segmentstore.contracts.SegmentProperties used on opensource projects

Example 41 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentMapper method submitToOperationLog.

/**
 * Submits a StreamSegmentMapOperation to the OperationLog. Upon completion, this operation
 * will have mapped the given Segment to a new internal Segment Id if none was provided in the given SegmentInfo.
 * If the given SegmentInfo already has a SegmentId set, then all efforts will be made to map that Segment with the
 * requested Segment Id.
 *
 * @param segmentInfo           The SegmentInfo for the StreamSegment to generate and persist.
 * @param parentStreamSegmentId If different from ContainerMetadata.NO_STREAM_SEGMENT_ID, the given streamSegmentInfo
 *                              will be mapped as a transaction. Otherwise, this will be registered as a standalone StreamSegment.
 * @param timeout               Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain the internal SegmentId that was assigned (or the
 * one supplied via SegmentInfo, if any). If the operation failed, then this Future will complete with that exception.
 */
private CompletableFuture<Long> submitToOperationLog(SegmentInfo segmentInfo, long parentStreamSegmentId, Duration timeout) {
    SegmentProperties properties = segmentInfo.getProperties();
    if (properties.isDeleted()) {
        // Stream does not exist. Fail the request with the appropriate exception.
        failAssignment(properties.getName(), new StreamSegmentNotExistsException("StreamSegment does not exist."));
        return Futures.failedFuture(new StreamSegmentNotExistsException(properties.getName()));
    }
    long existingSegmentId = this.containerMetadata.getStreamSegmentId(properties.getName(), true);
    if (isValidStreamSegmentId(existingSegmentId)) {
        // Looks like someone else beat us to it.
        completeAssignment(properties.getName(), existingSegmentId);
        return CompletableFuture.completedFuture(existingSegmentId);
    } else {
        StreamSegmentMapOperation op;
        if (isValidStreamSegmentId(parentStreamSegmentId)) {
            // Transaction.
            SegmentMetadata parentMetadata = this.containerMetadata.getStreamSegmentMetadata(parentStreamSegmentId);
            assert parentMetadata != null : "parentMetadata is null";
            op = new StreamSegmentMapOperation(parentStreamSegmentId, properties);
        } else {
            // Standalone StreamSegment.
            op = new StreamSegmentMapOperation(properties);
        }
        if (segmentInfo.getSegmentId() != ContainerMetadata.NO_STREAM_SEGMENT_ID) {
            op.setStreamSegmentId(segmentInfo.getSegmentId());
        }
        return this.durableLog.add(op, timeout).thenApply(seqNo -> completeAssignment(properties.getName(), op.getStreamSegmentId()));
    }
}

Example 42 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class SegmentAggregator method mergeWith.

/**
 * Merges the Transaction StreamSegment with given metadata into this one at the current offset.
 *
 * @param transactionMetadata The metadata of the Transaction StreamSegment to merge.
 * @param timer               Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the number of bytes that were merged into this
 * StreamSegment. If failed, the Future will contain the exception that caused it.
 */
private CompletableFuture<FlushResult> mergeWith(UpdateableSegmentMetadata transactionMetadata, MergeTransactionOperation mergeOp, TimeoutTimer timer) {
    if (transactionMetadata.isDeleted()) {
        return Futures.failedFuture(new DataCorruptionException(String.format("Attempted to merge with deleted Transaction segment '%s'.", transactionMetadata.getName())));
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "mergeWith", transactionMetadata.getId(), transactionMetadata.getName(), transactionMetadata.isSealedInStorage());
    FlushResult result = new FlushResult();
    if (!transactionMetadata.isSealedInStorage() || transactionMetadata.getLength() > transactionMetadata.getStorageLength()) {
        // Nothing to do. Given Transaction is not eligible for merger yet.
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return CompletableFuture.completedFuture(result);
    }
    AtomicLong mergedLength = new AtomicLong();
    return this.storage.getStreamSegmentInfo(transactionMetadata.getName(), timer.getRemaining()).thenAcceptAsync(transProperties -> {
        // Check that the Storage agrees with our metadata (if not, we have a problem ...)
        if (transProperties.getLength() != transactionMetadata.getStorageLength()) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' cannot be merged into parent '%s' because its metadata disagrees with the Storage. Metadata.StorageLength=%d, Storage.StorageLength=%d", transactionMetadata.getName(), this.metadata.getName(), transactionMetadata.getStorageLength(), transProperties.getLength())));
        }
        if (transProperties.getLength() != mergeOp.getLength()) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' cannot be merged into parent '%s' because the declared length in the operation disagrees with the Storage. Operation.Length=%d, Storage.StorageLength=%d", transactionMetadata.getName(), this.metadata.getName(), mergeOp.getLength(), transProperties.getLength())));
        }
        mergedLength.set(transProperties.getLength());
    }, this.executor).thenComposeAsync(v1 -> storage.concat(this.handle.get(), mergeOp.getStreamSegmentOffset(), transactionMetadata.getName(), timer.getRemaining()), this.executor).thenComposeAsync(v2 -> storage.getStreamSegmentInfo(this.metadata.getName(), timer.getRemaining()), this.executor).thenApplyAsync(segmentProperties -> {
        // We have processed a MergeTransactionOperation, pop the first operation off and decrement the counter.
        StorageOperation processedOperation = this.operations.removeFirst();
        assert processedOperation != null && processedOperation instanceof MergeTransactionOperation : "First outstanding operation was not a MergeTransactionOperation";
        assert ((MergeTransactionOperation) processedOperation).getTransactionSegmentId() == transactionMetadata.getId() : "First outstanding operation was a MergeTransactionOperation for the wrong Transaction id.";
        int newCount = this.mergeTransactionCount.decrementAndGet();
        assert newCount >= 0 : "Negative value for mergeTransactionCount";
        // Post-merger validation. Verify we are still in agreement with the storage.
        long expectedNewLength = this.metadata.getStorageLength() + mergedLength.get();
        if (segmentProperties.getLength() != expectedNewLength) {
            throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' was merged into parent '%s' but the parent segment has an unexpected StorageLength after the merger. Previous=%d, MergeLength=%d, Expected=%d, Actual=%d", transactionMetadata.getName(), this.metadata.getName(), segmentProperties.getLength(), mergedLength.get(), expectedNewLength, segmentProperties.getLength())));
        }
        updateMetadata(segmentProperties);
        updateMetadataForTransactionPostMerger(transactionMetadata);
        this.lastFlush.set(this.timer.getElapsed());
        result.withMergedBytes(mergedLength.get());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return result;
    }, this.executor).exceptionally(ex -> {
        Throwable realEx = Exceptions.unwrap(ex);
        if (realEx instanceof BadOffsetException || realEx instanceof StreamSegmentNotExistsException) {
            // We either attempted to write at an offset that already contained other data or the Transaction
            // Segment no longer exists. This can happen for a number of reasons, but we do not have enough
            // information here to determine why. We need to enter reconciliation mode, and hope for the best.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}

Example 43 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class SegmentAggregator method reconcile.

private CompletableFuture<FlushResult> reconcile(TimeoutTimer timer) {
    assert this.state.get() == AggregatorState.Reconciling : "reconcile cannot be called if state == " + this.state;
    ReconciliationState rc = this.reconciliationState.get();
    assert rc != null : "reconciliationState is null";
    SegmentProperties storageInfo = rc.getStorageInfo();
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "reconcile", rc);
    // Process each Operation in sequence, as long as its starting offset is less than ReconciliationState.getStorageInfo().getLength()
    FlushResult result = new FlushResult();
    AtomicBoolean exceededStorageLength = new AtomicBoolean(false);
    return Futures.loop(() -> this.operations.size() > 0 && !exceededStorageLength.get(), () -> {
        StorageOperation op = this.operations.getFirst();
        return reconcileOperation(op, storageInfo, timer).thenApply(partialFlushResult -> {
            if (op.getLastStreamSegmentOffset() >= storageInfo.getLength()) {
                // This operation crosses the boundary of StorageLength. It has been reconciled,
                // and as such it is the last operation that we need to inspect.
                exceededStorageLength.set(true);
            }
            log.info("{}: Reconciled {} ({}).", this.traceObjectId, op, partialFlushResult);
            return partialFlushResult;
        });
    }, result::withFlushResult, this.executor).thenApply(v -> {
        updateMetadata(storageInfo);
        this.reconciliationState.set(null);
        setState(AggregatorState.Writing);
        LoggerHelpers.traceLeave(log, this.traceObjectId, "reconcile", traceId, result);
        return result;
    });
}