Examples with StreamSegmentSealOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation used on opensource projects

Example 6 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class OperationLogTestBase method generateOperations.

// endregion
// region Operation Generation
/**
 * Generates a List of Log Operations that contains the following operations, in the "correct" order.
 * <ol>
 * <li> A set of StreamSegmentAppend Operations (based on the streamSegmentIds arg).
 * <li> A set of StreamSegmentSeal and MergeTransaction Operations (based on the TransactionIds and mergeTransactions arg).
 * <li> A set of StreamSegmentSeal Operations (based on the sealStreamSegments arg).
 * </ol>
 */
List<Operation> generateOperations(Collection<Long> streamSegmentIds, Map<Long, Long> transactionIds, int appendsPerStreamSegment, int metadataCheckpointsEvery, boolean mergeTransactions, boolean sealStreamSegments) {
    List<Operation> result = new ArrayList<>();
    // Add some appends.
    int appendId = 0;
    for (long streamSegmentId : streamSegmentIds) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(streamSegmentId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    for (long transactionId : transactionIds.keySet()) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(transactionId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    // Merge Transactions.
    if (mergeTransactions) {
        // Key = TransactionId, Value = Parent Id.
        transactionIds.entrySet().forEach(mapping -> {
            result.add(new StreamSegmentSealOperation(mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            result.add(new MergeTransactionOperation(mapping.getValue(), mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    // Seal the StreamSegments.
    if (sealStreamSegments) {
        streamSegmentIds.forEach(streamSegmentId -> {
            result.add(new StreamSegmentSealOperation(streamSegmentId));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    return result;
}

Example 7 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessMergeTransaction.

// endregion
// region MergeTransactionOperation
/**
 * Tests the preProcess method with MergeTransactionOperations.
 * Scenarios:
 * * Recovery/non-recovery mode
 * * Target StreamSegment is sealed
 * * Target StreamSegment is a Transaction
 * * Transaction StreamSegment is already merged
 * * Transaction StreamSegment is not sealed
 */
@Test
public void testPreProcessMergeTransaction() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // When everything is OK (recovery mode).
    MergeTransactionOperation recoveryMergeOp = createMerge();
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    AssertExtensions.assertThrows("preProcess(Merge) handled an operation with no Transaction StreamSegment Length set.", () -> txn1.preProcessOperation(createMerge()), ex -> ex instanceof MetadataUpdateException);
    // In recovery mode, the updater does not set the length; it just validates that it has one.
    recoveryMergeOp.setLength(metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).getLength());
    txn1.preProcessOperation(recoveryMergeOp);
    AssertExtensions.assertLessThan("Unexpected Target StreamSegmentOffset after call to preProcess in recovery mode.", 0, recoveryMergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(recoveryMergeOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When everything is OK (non-recovery mode).
    MergeTransactionOperation mergeOp = createMerge();
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mergeOp);
    Assert.assertEquals("Unexpected Transaction StreamSegmentLength after call to preProcess in non-recovery mode.", SEALED_TRANSACTION_LENGTH, mergeOp.getLength());
    Assert.assertEquals("Unexpected Target StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, mergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(mergeOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in non-recovery mode.", txn2.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in non-recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When Target StreamSegment is sealed.
    StreamSegmentSealOperation sealTargetOp = createSeal();
    txn2.preProcessOperation(sealTargetOp);
    txn2.acceptOperation(sealTargetOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is sealed.", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentSealedException);
    // Rollback the seal
    txn2.clear();
    // When Target StreamSegment is a Transaction.
    MergeTransactionOperation mergeToTransactionOp = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEALED_TRANSACTION_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is a Transaction.", () -> txn2.preProcessOperation(mergeToTransactionOp), ex -> ex instanceof MetadataUpdateException);
    // When Transaction is not sealed.
    MergeTransactionOperation mergeNonSealed = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEGMENT_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is not sealed.", () -> txn2.preProcessOperation(mergeNonSealed), ex -> ex instanceof StreamSegmentNotSealedException);
    // When Transaction is already merged.
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in transaction).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
    txn2.commit(metadata);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in metadata).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
}

Example 8 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAcceptStreamSegmentSeal.

/**
 * Tests the accept method with StreamSegmentSeal operations.
 */
@Test
public void testAcceptStreamSegmentSeal() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // Set some attributes.
    val segmentAttributes = createAttributes();
    segmentAttributes.put(Attributes.CREATION_TIME, 1L);
    UpdateableSegmentMetadata segmentMetadata = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    segmentMetadata.updateAttributes(segmentAttributes);
    val txn = createUpdateTransaction(metadata);
    StreamSegmentSealOperation sealOp = createSeal();
    // When no pre-process has happened.
    AssertExtensions.assertThrows("Unexpected behavior from acceptOperation() when no pre-processing was made.", () -> txn.acceptOperation(sealOp), ex -> ex instanceof MetadataUpdateException);
    Assert.assertFalse("acceptOperation updated the transaction even if it threw an exception.", txn.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("acceptOperation updated the metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When all is good.
    txn.preProcessOperation(sealOp);
    txn.acceptOperation(sealOp);
    Assert.assertTrue("acceptOperation did not update the transaction.", txn.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("acceptOperation updated the metadata.", segmentMetadata.isSealed());
    txn.commit(metadata);
    // Check attributes.
    // All dynamic attributes should be removed.
    segmentAttributes.keySet().removeIf(Attributes::isDynamic);
    SegmentMetadataComparer.assertSameAttributes("Unexpected set of attributes after commit.", segmentAttributes, segmentMetadata);
}

Example 9 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class SegmentAggregatorTests method testAddWithBadInput.

/**
 * Tests the add() method with invalid arguments.
 */
@Test
public void testAddWithBadInput() throws Exception {
    final long badTransactionId = 12345;
    final long badParentId = 56789;
    final String badParentName = "Foo_Parent";
    final String badTransactionName = "Foo_Transaction";
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG);
    // We only needs one Transaction for this test.
    SegmentAggregator transactionAggregator = context.transactionAggregators[0];
    SegmentMetadata transactionMetadata = transactionAggregator.getMetadata();
    context.storage.create(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join();
    context.storage.create(transactionMetadata.getName(), TIMEOUT).join();
    context.segmentAggregator.initialize(TIMEOUT).join();
    transactionAggregator.initialize(TIMEOUT).join();
    // Create 2 more segments that can be used to verify MergeTransactionOperation.
    context.containerMetadata.mapStreamSegmentId(badParentName, badParentId);
    UpdateableSegmentMetadata badTransactionMetadata = context.containerMetadata.mapStreamSegmentId(badTransactionName, badTransactionId, badParentId);
    badTransactionMetadata.setLength(0);
    badTransactionMetadata.setStorageLength(0);
    context.storage.create(badTransactionMetadata.getName(), TIMEOUT).join();
    // 1. MergeTransactionOperation
    // 1a.Verify that MergeTransactionOperation cannot be added to the Transaction segment.
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation on the Transaction segment.", () -> transactionAggregator.add(generateSimpleMergeTransaction(transactionMetadata.getId(), context)), ex -> ex instanceof IllegalArgumentException);
    // 1b. Verify that MergeTransactionOperation has the right parent.
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation on the parent for a Transaction that did not have it as a parent.", () -> transactionAggregator.add(generateSimpleMergeTransaction(badTransactionId, context)), ex -> ex instanceof IllegalArgumentException);
    // 2. StreamSegmentSealOperation.
    // 2a. Verify we cannot add a StreamSegmentSealOperation if the segment is not sealed yet.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentSealOperation for a non-sealed segment.", () -> {
        @Cleanup SegmentAggregator badTransactionAggregator = new SegmentAggregator(badTransactionMetadata, context.dataSource, context.storage, DEFAULT_CONFIG, context.timer, executorService());
        badTransactionAggregator.initialize(TIMEOUT).join();
        badTransactionAggregator.add(generateSimpleSeal(badTransactionId, context));
    }, ex -> ex instanceof DataCorruptionException);
    // 2b. Verify that nothing is allowed after Seal (after adding one append to and sealing the Transaction Segment).
    StorageOperation transactionAppend1 = generateAppendAndUpdateMetadata(0, transactionMetadata.getId(), context);
    transactionAggregator.add(transactionAppend1);
    transactionAggregator.add(generateSealAndUpdateMetadata(transactionMetadata.getId(), context));
    AssertExtensions.assertThrows("add() allowed operation after seal.", () -> transactionAggregator.add(generateSimpleAppend(transactionMetadata.getId(), context)), ex -> ex instanceof DataCorruptionException);
    // 3. CachedStreamSegmentAppendOperation.
    final StorageOperation parentAppend1 = generateAppendAndUpdateMetadata(0, SEGMENT_ID, context);
    // 3a. Verify we cannot add StreamSegmentAppendOperations.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentAppendOperation.", () -> {
        // We have the correct offset, but we did not increase the Length.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(parentAppend1.getStreamSegmentId(), parentAppend1.getStreamSegmentOffset(), new byte[(int) parentAppend1.getLength()], null);
        context.segmentAggregator.add(badAppend);
    }, ex -> ex instanceof IllegalArgumentException);
    // Add this one append to the parent (nothing unusual here); we'll use this for the next tests.
    context.segmentAggregator.add(parentAppend1);
    // 3b. Verify we cannot add anything beyond the DurableLogOffset (offset or offset+length).
    AssertExtensions.assertThrows("add() allowed an operation beyond the DurableLogOffset (offset).", () -> {
        // We have the correct offset, but we did not increase the Length.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badAppend));
    }, ex -> ex instanceof DataCorruptionException);
    ((UpdateableSegmentMetadata) context.segmentAggregator.getMetadata()).setLength(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength() + 1);
    AssertExtensions.assertThrows("add() allowed an operation beyond the DurableLogOffset (offset+length).", () -> {
        // We have the correct offset, but we the append exceeds the Length by 1 byte.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badAppend));
    }, ex -> ex instanceof DataCorruptionException);
    // 3c. Verify contiguity (offsets - we cannot have gaps in the data).
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too small).", () -> {
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(0);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too large).", () -> {
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength() + 1);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too large, but no pending operations).", () -> {
        @Cleanup SegmentAggregator badTransactionAggregator = new SegmentAggregator(badTransactionMetadata, context.dataSource, context.storage, DEFAULT_CONFIG, context.timer, executorService());
        badTransactionMetadata.setLength(100);
        badTransactionAggregator.initialize(TIMEOUT).join();
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(1);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    // 4. Verify Segment Id match.
    AssertExtensions.assertThrows("add() allowed an Append operation with wrong Segment Id.", () -> {
        StreamSegmentAppendOperation badIdAppend = new StreamSegmentAppendOperation(Integer.MAX_VALUE, "foo".getBytes(), null);
        badIdAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badIdAppend));
    }, ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("add() allowed a StreamSegmentSealOperation with wrong SegmentId.", () -> {
        StreamSegmentSealOperation badIdSeal = new StreamSegmentSealOperation(Integer.MAX_VALUE);
        badIdSeal.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(badIdSeal);
    }, ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation with wrong SegmentId.", () -> {
        MergeTransactionOperation badIdMerge = new MergeTransactionOperation(Integer.MAX_VALUE, transactionMetadata.getId());
        badIdMerge.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        badIdMerge.setLength(1);
        context.segmentAggregator.add(badIdMerge);
    }, ex -> ex instanceof IllegalArgumentException);
    // 5. Truncations.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentTruncateOperation with a truncation offset beyond the one in the metadata.", () -> {
        StreamSegmentTruncateOperation op = new StreamSegmentTruncateOperation(SEGMENT_ID, 10);
        op.setSequenceNumber(context.containerMetadata.nextOperationSequenceNumber());
        context.segmentAggregator.add(op);
    }, ex -> ex instanceof DataCorruptionException);
}

Example 10 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class StorageWriterTests method sealSegments.

private void sealSegments(Collection<Long> segmentIds, TestContext context) {
    for (long segmentId : segmentIds) {
        UpdateableSegmentMetadata segmentMetadata = context.metadata.getStreamSegmentMetadata(segmentId);
        segmentMetadata.markSealed();
        StreamSegmentSealOperation sealOp = new StreamSegmentSealOperation(segmentId);
        sealOp.setStreamSegmentOffset(segmentMetadata.getLength());
        context.dataSource.add(sealOp);
    }
}