Examples with MergeSegmentOperation io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation used on opensource projects

Example 26 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class SegmentAggregator method mergeWith.

/**
 * Merges the Transaction StreamSegment with given metadata into this one at the current offset.
 *
 * @param transactionMetadata The metadata of the Transaction StreamSegment to merge.
 * @param timer               Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the number of bytes that were merged into this
 * StreamSegment. If failed, the Future will contain the exception that caused it.
 */
private CompletableFuture<WriterFlushResult> mergeWith(UpdateableSegmentMetadata transactionMetadata, MergeSegmentOperation mergeOp, TimeoutTimer timer) {
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "mergeWith", transactionMetadata.getId(), transactionMetadata.getName(), transactionMetadata.isSealedInStorage());
    boolean emptySourceSegment = transactionMetadata.getLength() == 0;
    if (transactionMetadata.isDeleted() && !emptySourceSegment) {
        // We came across a deleted source segment that had some data. We need to begin a reconciliation to figure out
        // the actual state of the segments in Storage.
        setState(AggregatorState.ReconciliationNeeded);
        return Futures.failedFuture(new StreamSegmentNotExistsException(transactionMetadata.getName()));
    }
    WriterFlushResult result = new WriterFlushResult();
    CompletableFuture<SegmentProperties> merge;
    if (emptySourceSegment) {
        // We came across a deleted source segment which had no data. No point in attempting to do anything, as any
        // operation involving this segment will complain about it not being there.
        log.warn("{}: Not applying '{}' because source segment is missing or empty.", this.traceObjectId, mergeOp);
        merge = CompletableFuture.completedFuture(this.metadata);
    } else if (!transactionMetadata.isSealedInStorage() || transactionMetadata.getLength() > transactionMetadata.getStorageLength()) {
        // Nothing to do. Given Transaction is not eligible for merger yet.
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return CompletableFuture.completedFuture(result);
    } else {
        merge = mergeInStorage(transactionMetadata, mergeOp, timer);
    }
    // need to make AttributeAggregator aware of merging segments.
    return merge.thenAcceptAsync(segmentProperties -> mergeCompleted(segmentProperties, transactionMetadata, mergeOp), this.executor).thenComposeAsync(v -> this.dataSource.deleteAllAttributes(transactionMetadata, timer.getRemaining()), this.executor).thenApply(v -> {
        this.lastFlush.set(this.timer.getElapsed());
        result.withMergedBytes(mergeOp.getLength());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeWith", traceId, result);
        return result;
    }).exceptionally(ex -> {
        Throwable realEx = Exceptions.unwrap(ex);
        if (realEx instanceof BadOffsetException || realEx instanceof StreamSegmentNotExistsException) {
            // We either attempted to write at an offset that already contained other data or the Transaction
            // Segment no longer exists. This can happen for a number of reasons, but we do not have enough
            // information here to determine why. We need to enter reconciliation mode, and hope for the best.
            setState(AggregatorState.ReconciliationNeeded);
        }
        // Rethrow all exceptions.
        throw new CompletionException(ex);
    });
}

Example 27 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class DurableDataLogRepairCommand method createUserDefinedOperation.

/**
 * Guides the user to generate a new {@link Operation} that will eventually modify the Original Log.
 *
 * @return New {@link Operation} to be added in the Original Log.
 */
@VisibleForTesting
Operation createUserDefinedOperation() {
    Operation result;
    final String operations = "[DeleteSegmentOperation|MergeSegmentOperation|MetadataCheckpointOperation|" + "StorageMetadataCheckpointOperation|StreamSegmentAppendOperation|StreamSegmentMapOperation|" + "StreamSegmentSealOperation|StreamSegmentTruncateOperation|UpdateAttributesOperation]";
    switch(getStringUserInput("Type one of the following Operations to instantiate: " + operations)) {
        case "DeleteSegmentOperation":
            long segmentId = getLongUserInput("Input Segment Id for DeleteSegmentOperation:");
            result = new DeleteSegmentOperation(segmentId);
            long offset = getLongUserInput("Input Segment Offset for DeleteSegmentOperation:");
            ((DeleteSegmentOperation) result).setStreamSegmentOffset(offset);
            break;
        case "MergeSegmentOperation":
            long targetSegmentId = getLongUserInput("Input Target Segment Id for MergeSegmentOperation:");
            long sourceSegmentId = getLongUserInput("Input Source Segment Id for MergeSegmentOperation:");
            result = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, createAttributeUpdateCollection());
            offset = getLongUserInput("Input Segment Offset for MergeSegmentOperation:");
            ((MergeSegmentOperation) result).setStreamSegmentOffset(offset);
            break;
        case "MetadataCheckpointOperation":
            result = new MetadataCheckpointOperation();
            ((MetadataCheckpointOperation) result).setContents(createOperationContents());
            break;
        case "StorageMetadataCheckpointOperation":
            result = new StorageMetadataCheckpointOperation();
            ((StorageMetadataCheckpointOperation) result).setContents(createOperationContents());
            break;
        case "StreamSegmentAppendOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentAppendOperation:");
            offset = getLongUserInput("Input Segment Offset for StreamSegmentAppendOperation:");
            result = new StreamSegmentAppendOperation(segmentId, offset, createOperationContents(), createAttributeUpdateCollection());
            break;
        case "StreamSegmentMapOperation":
            result = new StreamSegmentMapOperation(createSegmentProperties());
            break;
        case "StreamSegmentSealOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentSealOperation:");
            result = new StreamSegmentSealOperation(segmentId);
            offset = getLongUserInput("Input Segment Offset for StreamSegmentSealOperation:");
            ((StreamSegmentSealOperation) result).setStreamSegmentOffset(offset);
            break;
        case "StreamSegmentTruncateOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentTruncateOperation:");
            offset = getLongUserInput("Input Offset for StreamSegmentTruncateOperation:");
            result = new StreamSegmentTruncateOperation(segmentId, offset);
            break;
        case "UpdateAttributesOperation":
            segmentId = getLongUserInput("Input Segment Id for UpdateAttributesOperation:");
            result = new UpdateAttributesOperation(segmentId, createAttributeUpdateCollection());
            break;
        default:
            output("Invalid operation, please select one of " + operations);
            throw new UnsupportedOperationException();
    }
    return result;
}

Example 28 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessMergeSegment.

// endregion
// region MergeSegmentOperation
/**
 * Tests the preProcess method with MergeTransactionOperations.
 * Scenarios:
 * * Recovery/non-recovery mode
 * * Target StreamSegment is sealed
 * * Target StreamSegment is a Transaction
 * * Source StreamSegment is already merged
 * * Source StreamSegment is not sealed
 */
@Test
public void testPreProcessMergeSegment() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // When everything is OK (recovery mode).
    MergeSegmentOperation recoveryMergeOp = createMerge();
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    AssertExtensions.assertThrows("preProcess(Merge) handled an operation with no Transaction StreamSegment Length set.", () -> txn1.preProcessOperation(createMerge()), ex -> ex instanceof MetadataUpdateException);
    // In recovery mode, the updater does not set the length; it just validates that it has one.
    recoveryMergeOp.setLength(metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).getLength());
    txn1.preProcessOperation(recoveryMergeOp);
    AssertExtensions.assertLessThan("Unexpected Target StreamSegmentOffset after call to preProcess in recovery mode.", 0, recoveryMergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(recoveryMergeOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    // When everything is OK (non-recovery mode).
    MergeSegmentOperation mergeOp = createMerge();
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mergeOp);
    Assert.assertEquals("Unexpected Transaction StreamSegmentLength after call to preProcess in non-recovery mode.", SEALED_SOURCE_LENGTH, mergeOp.getLength());
    Assert.assertEquals("Unexpected Target StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, mergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(mergeOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in non-recovery mode.", txn2.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in non-recovery mode.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    // When Target StreamSegment is sealed.
    StreamSegmentSealOperation sealTargetOp = createSeal();
    txn2.preProcessOperation(sealTargetOp);
    txn2.acceptOperation(sealTargetOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is sealed.", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentSealedException);
    // Rollback the seal
    txn2.clear();
    // When Transaction is not sealed.
    MergeSegmentOperation mergeNonSealed = new MergeSegmentOperation(NOTSEALED_SOURCE_ID, SEGMENT_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is not sealed.", () -> txn2.preProcessOperation(mergeNonSealed), ex -> ex instanceof StreamSegmentNotSealedException);
    // When Transaction is already merged.
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in transaction).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
    txn2.commit(metadata);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in metadata).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
}

Example 29 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testCommit.

private void testCommit(UpdateableContainerMetadata baseMetadata, UpdateableContainerMetadata targetMetadata) throws Exception {
    // Create a few appends, merge a Transaction and seal the target Segment. Verify all changes have been applied after
    // a call to commit().
    int appendCount = 500;
    ArrayList<StorageOperation> operations = new ArrayList<>();
    int appendAttributeCount = 0;
    for (int i = 0; i < appendCount; i++) {
        StreamSegmentAppendOperation op = createAppendNoOffset();
        operations.add(op);
        appendAttributeCount += op.getAttributeUpdates().size();
    }
    operations.add(createMerge());
    operations.add(createSeal());
    // This is a change we make only to the baseMetadata. This should not propagate to the targetMetadata.
    baseMetadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(Collections.singletonMap(Attributes.CREATION_TIME, 0L));
    val txn = createUpdateTransaction(baseMetadata);
    long expectedLastUsedParent = -1;
    long expectedLastUsedTransaction = -1;
    long seqNo = 0;
    for (StorageOperation op : operations) {
        txn.preProcessOperation(op);
        op.setSequenceNumber(++seqNo);
        txn.acceptOperation(op);
        if (op.getStreamSegmentId() == SEGMENT_ID) {
            expectedLastUsedParent = op.getSequenceNumber();
        }
        if (op instanceof MergeSegmentOperation) {
            expectedLastUsedParent = op.getSequenceNumber();
            expectedLastUsedTransaction = op.getSequenceNumber();
        }
    }
    txn.commit(targetMetadata);
    Assert.assertEquals("commit() seems to have modified the metadata sequence number while not in recovery mode.", ContainerMetadata.INITIAL_OPERATION_SEQUENCE_NUMBER, targetMetadata.nextOperationSequenceNumber() - 1);
    long expectedLength = SEGMENT_LENGTH + appendCount * DEFAULT_APPEND_DATA.getLength() + SEALED_SOURCE_LENGTH;
    SegmentMetadata parentMetadata = targetMetadata.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in metadata after commit.", expectedLength, parentMetadata.getLength());
    Assert.assertTrue("Unexpected value for isSealed in metadata after commit.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Target after commit.", expectedLastUsedParent, parentMetadata);
    SegmentMetadata transactionMetadata = targetMetadata.getStreamSegmentMetadata(SEALED_SOURCE_ID);
    Assert.assertTrue("Unexpected value for isSealed in Source metadata after commit.", transactionMetadata.isSealed());
    Assert.assertTrue("Unexpected value for isMerged in Source metadata after commit.", transactionMetadata.isMerged());
    int expectedParentAttributeCount = appendAttributeCount + (baseMetadata == targetMetadata ? 1 : 0) + 1;
    Assert.assertEquals("Unexpected number of attributes for Target segment.", expectedParentAttributeCount, parentMetadata.getAttributes().size());
    Assert.assertEquals(DEFAULT_TYPE.getValue(), (long) parentMetadata.getAttributes().get(Attributes.ATTRIBUTE_SEGMENT_TYPE));
    Assert.assertEquals("Unexpected number of attributes for Source.", 1, transactionMetadata.getAttributes().size());
    Assert.assertEquals(DEFAULT_TYPE.getValue(), (long) transactionMetadata.getAttributes().get(Attributes.ATTRIBUTE_SEGMENT_TYPE));
    checkLastKnownSequenceNumber("Unexpected lastUsed for Source after commit.", expectedLastUsedTransaction, transactionMetadata);
}