Examples with MergeSegmentOperation io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation used on opensource projects

Example 11 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class SegmentAggregator method mergeIfNecessary.

/**
 * Executes a merger of a Transaction StreamSegment into this one.
 * Conditions for merger:
 * <ul>
 * <li> This StreamSegment is stand-alone (not a Transaction).
 * <li> The next outstanding operation is a MergeSegmentOperation for a Transaction StreamSegment of this StreamSegment.
 * <li> The StreamSegment to merge is not deleted, it is sealed and is fully flushed to Storage.
 * </ul>
 * Effects of the merger:
 * <ul> The entire contents of the given Transaction StreamSegment will be concatenated to this StreamSegment as one unit.
 * <li> The metadata for this StreamSegment will be updated to reflect the new length of this StreamSegment.
 * <li> The given Transaction Segment will cease to exist.
 * </ul>
 * <p>
 * Note that various other data integrity checks are done pre and post merger as part of this operation which are meant
 * to ensure the StreamSegment is not in a corrupted state.
 *
 * @param flushResult The flush result from the previous chained operation.
 * @param timer       Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the number of bytes that were merged into this
 * StreamSegment. If failed, the Future will contain the exception that caused it.
 */
private CompletableFuture<WriterFlushResult> mergeIfNecessary(WriterFlushResult flushResult, TimeoutTimer timer) {
    ensureInitializedAndNotClosed();
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "mergeIfNecessary");
    StorageOperation first = this.operations.getFirst();
    if (first == null || !(first instanceof MergeSegmentOperation)) {
        // Either no operation or first operation is not a MergeTransaction. Nothing to do.
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeIfNecessary", traceId, flushResult);
        return CompletableFuture.completedFuture(flushResult);
    }
    MergeSegmentOperation mergeSegmentOperation = (MergeSegmentOperation) first;
    UpdateableSegmentMetadata transactionMetadata = this.dataSource.getStreamSegmentMetadata(mergeSegmentOperation.getSourceSegmentId());
    return mergeWith(transactionMetadata, mergeSegmentOperation, timer).thenApply(mergeResult -> {
        flushResult.withFlushResult(mergeResult);
        LoggerHelpers.traceLeave(log, this.traceObjectId, "mergeIfNecessary", traceId, flushResult);
        return flushResult;
    });
}

Example 12 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessStreamSegmentSeal.

// endregion
// region StreamSegmentSealOperation
/**
 * Tests the preProcess method with StreamSegmentSeal operations.
 * Scenarios:
 * * Recovery Mode
 * * Non-recovery mode
 * * StreamSegment is Merged (both in-transaction and in-metadata)
 * * StreamSegment is Sealed (both in-transaction and in-metadata)
 */
@Test
public void testPreProcessStreamSegmentSeal() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    StreamSegmentSealOperation sealOp = createSeal();
    // When everything is OK (in recovery mode) - nothing should change.
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    txn1.preProcessOperation(sealOp);
    AssertExtensions.assertLessThan("Unexpected StreamSegmentLength after call to preProcess in recovery mode.", 0, sealOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(sealOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Seal) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("preProcess(Seal) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When everything is OK (no recovery mode).
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(sealOp);
    Assert.assertEquals("Unexpected StreamSegmentLength after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, sealOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(sealOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Seal) seems to have changed the Updater internal state.", txn2.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("preProcess(Seal) seems to have changed the metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When StreamSegment is merged (via transaction).
    StreamSegmentSealOperation transactionSealOp = new StreamSegmentSealOperation(SEALED_SOURCE_ID);
    MergeSegmentOperation mergeOp = createMerge();
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    Assert.assertFalse("Transaction should not be merged in metadata (yet).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is merged (in transaction).", () -> txn2.preProcessOperation(transactionSealOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is merged (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("Transaction should have been merged in metadata.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is merged (in metadata).", () -> txn2.preProcessOperation(transactionSealOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is sealed (via transaction).
    txn2.acceptOperation(sealOp);
    Assert.assertFalse("StreamSegment should not be sealed in metadata (yet).", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is sealed (in transaction).", () -> txn2.preProcessOperation(createSeal()), ex -> ex instanceof StreamSegmentSealedException);
    // When StreamSegment is sealed (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("StreamSegment should have been sealed in metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is sealed (in metadata).", () -> txn2.preProcessOperation(createSeal()), ex -> ex instanceof StreamSegmentSealedException);
}

Example 13 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAcceptMergeSegment.

/**
 * Tests the accept method with MergeTransactionOperations.
 */
@Test
public void testAcceptMergeSegment() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    MergeSegmentOperation mergeOp = createMerge();
    // When no pre-process has happened
    AssertExtensions.assertThrows("Unexpected behavior from acceptOperation() when no pre-processing was made.", () -> txn.acceptOperation(mergeOp), ex -> ex instanceof MetadataUpdateException);
    Assert.assertEquals("acceptOperation updated the transaction even if it threw an exception (target segment).", SEGMENT_LENGTH, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata (target segment).", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // This would naturally happen in case of a failure, so we need to simulate this here too.
    txn.clear();
    // When all is good.
    txn.preProcessOperation(mergeOp);
    txn.acceptOperation(mergeOp);
    Assert.assertTrue("acceptOperation did not update the transaction(Transaction).", txn.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertFalse("acceptOperation updated the metadata (Transaction).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertEquals("acceptOperation did not update the transaction.", SEGMENT_LENGTH + SEALED_SOURCE_LENGTH, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
}

Example 14 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessStreamSegmentAppend.

// region StreamSegmentAppendOperation
/**
 * Tests the preProcess method with StreamSegmentAppend operations.
 * Scenarios:
 * * Recovery Mode
 * * Non-recovery mode
 * * StreamSegment is Merged (both in-transaction and in-metadata)
 * * StreamSegment is Sealed (both in-transaction and in-metadata)
 */
@Test
public void testPreProcessStreamSegmentAppend() throws Exception {
    val metadata = createMetadata();
    StreamSegmentAppendOperation appendOp = createAppendNoOffset();
    // When everything is OK (in recovery mode) - nothing should change.
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    txn1.preProcessOperation(appendOp);
    AssertExtensions.assertLessThan("Unexpected StreamSegmentOffset after call to preProcess in recovery mode.", 0, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state in recovery mode.", SEGMENT_LENGTH, txn1.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata in recovery mode.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When everything is OK (no recovery mode).
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", SEGMENT_LENGTH, txn2.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When StreamSegment is merged (via transaction).
    StreamSegmentAppendOperation transactionAppendOp = new StreamSegmentAppendOperation(SEALED_SOURCE_ID, DEFAULT_APPEND_DATA, null);
    MergeSegmentOperation mergeOp = createMerge();
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    Assert.assertFalse("Transaction should not be merged in metadata (yet).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in transaction).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is merged (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("Transaction should have been merged in metadata.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in metadata).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is sealed (via transaction).
    StreamSegmentSealOperation sealOp = createSeal();
    txn2.preProcessOperation(sealOp);
    txn2.acceptOperation(sealOp);
    Assert.assertFalse("StreamSegment should not be sealed in metadata (yet).", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in transaction).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
    // When StreamSegment is sealed (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("StreamSegment should have been sealed in metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in metadata).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
}

Example 15 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class StorageWriterTests method mergeTransactions.

private void mergeTransactions(Iterable<Long> transactionIds, HashMap<Long, ByteArrayOutputStream> segmentContents, TestContext context) {
    for (long transactionId : transactionIds) {
        UpdateableSegmentMetadata transactionMetadata = context.metadata.getStreamSegmentMetadata(transactionId);
        UpdateableSegmentMetadata parentMetadata = context.metadata.getStreamSegmentMetadata(context.transactionIds.get(transactionMetadata.getId()));
        Assert.assertFalse("Transaction already merged", transactionMetadata.isMerged());
        Assert.assertTrue("Transaction not sealed prior to merger", transactionMetadata.isSealed());
        Assert.assertFalse("Parent is sealed already merged", parentMetadata.isSealed());
        // Create the Merge Op
        MergeSegmentOperation op = new MergeSegmentOperation(parentMetadata.getId(), transactionMetadata.getId());
        op.setLength(transactionMetadata.getLength());
        op.setStreamSegmentOffset(parentMetadata.getLength());
        // Update metadata
        parentMetadata.setLength(parentMetadata.getLength() + transactionMetadata.getLength());
        transactionMetadata.markMerged();
        // Process the merge op
        context.dataSource.add(op);
        try {
            segmentContents.get(parentMetadata.getId()).write(segmentContents.get(transactionMetadata.getId()).toByteArray());
        } catch (IOException ex) {
            throw new AssertionError(ex);
        }
        segmentContents.remove(transactionId);
    }
}