Examples with StreamSegmentAppendOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation used on opensource projects

Example 16 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class OperationLogTestBase method generateOperations.

// endregion
// region Operation Generation
/**
 * Generates a List of Log Operations that contains the following operations, in the "correct" order.
 * <ol>
 * <li> A set of StreamSegmentAppend Operations (based on the streamSegmentIds arg).
 * <li> A set of StreamSegmentSeal and MergeTransaction Operations (based on the TransactionIds and mergeTransactions arg).
 * <li> A set of StreamSegmentSeal Operations (based on the sealStreamSegments arg).
 * </ol>
 */
List<Operation> generateOperations(Collection<Long> streamSegmentIds, Map<Long, Long> transactionIds, int appendsPerStreamSegment, int metadataCheckpointsEvery, boolean mergeTransactions, boolean sealStreamSegments) {
    List<Operation> result = new ArrayList<>();
    // Add some appends.
    int appendId = 0;
    for (long streamSegmentId : streamSegmentIds) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(streamSegmentId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    for (long transactionId : transactionIds.keySet()) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(transactionId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    // Merge Transactions.
    if (mergeTransactions) {
        // Key = TransactionId, Value = Parent Id.
        transactionIds.entrySet().forEach(mapping -> {
            result.add(new StreamSegmentSealOperation(mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            result.add(new MergeTransactionOperation(mapping.getValue(), mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    // Seal the StreamSegments.
    if (sealStreamSegments) {
        streamSegmentIds.forEach(streamSegmentId -> {
            result.add(new StreamSegmentSealOperation(streamSegmentId));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    return result;
}

Example 17 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class OperationMetadataUpdaterTests method recordAppend.

private void recordAppend(long segmentId, int length, OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception {
    byte[] data = new byte[length];
    val attributeUpdates = Arrays.asList(new AttributeUpdate(Attributes.CREATION_TIME, AttributeUpdateType.Replace, NEXT_ATTRIBUTE_VALUE.get()), new AttributeUpdate(Attributes.EVENT_COUNT, AttributeUpdateType.Accumulate, NEXT_ATTRIBUTE_VALUE.get()));
    val op = new StreamSegmentAppendOperation(segmentId, data, attributeUpdates);
    process(op, updater);
    if (referenceMetadata != null) {
        val rsm = referenceMetadata.getStreamSegmentMetadata(segmentId);
        rsm.setLength(rsm.getLength() + length);
        val attributes = new HashMap<UUID, Long>();
        op.getAttributeUpdates().forEach(au -> attributes.put(au.getAttributeId(), au.getValue()));
        rsm.updateAttributes(attributes);
    }
}

Example 18 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAcceptStreamSegmentAppend.

/**
 * Tests the accept method with StreamSegmentAppend operations.
 */
@Test
public void testAcceptStreamSegmentAppend() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    StreamSegmentAppendOperation appendOp = createAppendNoOffset();
    // When no pre-process has happened.
    AssertExtensions.assertThrows("Unexpected behavior from acceptOperation() when no pre-processing was made.", () -> txn.acceptOperation(appendOp), ex -> ex instanceof MetadataUpdateException);
    Assert.assertEquals("acceptOperation updated the transaction even if it threw an exception.", SEGMENT_LENGTH, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When all is good.
    txn.preProcessOperation(appendOp);
    txn.acceptOperation(appendOp);
    Assert.assertEquals("acceptOperation did not update the transaction.", SEGMENT_LENGTH + appendOp.getData().length, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
}

Example 19 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testRollback.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to rollback all outstanding changes.
 */
@Test
public void testRollback() throws Exception {
    // Create a couple of operations, commit them, and then create a few more appends, merge a Transaction and seal the parent stream.
    // Then call rollback(); verify no changes have been applied after the call to commit().
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    StreamSegmentAppendOperation committedAppend = createAppendNoOffset();
    txn.preProcessOperation(committedAppend);
    txn.acceptOperation(committedAppend);
    // This is the last extent of the modifications to the metadata.
    txn.commit(metadata);
    int appendCount = 500;
    ArrayList<StorageOperation> operations = new ArrayList<>();
    for (int i = 0; i < appendCount; i++) {
        operations.add(createAppendNoOffset());
    }
    operations.add(createMerge());
    operations.add(createSeal());
    long seqNo = 0;
    for (StorageOperation op : operations) {
        txn.preProcessOperation(op);
        op.setSequenceNumber(++seqNo);
        txn.acceptOperation(op);
    }
    txn.clear();
    long expectedLength = SEGMENT_LENGTH + DEFAULT_APPEND_DATA.length;
    // Verify metadata is untouched and that the updater has truly rolled back.
    SegmentMetadata parentMetadata = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in metadata after rollback.", expectedLength, parentMetadata.getLength());
    Assert.assertFalse("Unexpected value for isSealed in metadata after rollback.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Parent after rollback.", 0, parentMetadata);
    SegmentMetadata transactionMetadata = metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID);
    Assert.assertFalse("Unexpected value for isMerged in transaction segment metadata after rollback.", transactionMetadata.isMerged());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Transaction segment after rollback.", 0, transactionMetadata);
    // Now the updater
    parentMetadata = txn.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in transaction after rollback.", expectedLength, parentMetadata.getLength());
    Assert.assertFalse("Unexpected value for isSealed in transaction after rollback.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Parent (txn) after rollback.", 0, parentMetadata);
    transactionMetadata = txn.getStreamSegmentMetadata(SEALED_TRANSACTION_ID);
    Assert.assertFalse("Unexpected value for isMerged in transaction segment in update transaction after rollback.", transactionMetadata.isMerged());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Transaction segment in update transaction after rollback.", 0, transactionMetadata);
}

Example 20 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class SegmentAggregatorTests method testAddWithBadInput.

/**
 * Tests the add() method with invalid arguments.
 */
@Test
public void testAddWithBadInput() throws Exception {
    final long badTransactionId = 12345;
    final long badParentId = 56789;
    final String badParentName = "Foo_Parent";
    final String badTransactionName = "Foo_Transaction";
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG);
    // We only needs one Transaction for this test.
    SegmentAggregator transactionAggregator = context.transactionAggregators[0];
    SegmentMetadata transactionMetadata = transactionAggregator.getMetadata();
    context.storage.create(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join();
    context.storage.create(transactionMetadata.getName(), TIMEOUT).join();
    context.segmentAggregator.initialize(TIMEOUT).join();
    transactionAggregator.initialize(TIMEOUT).join();
    // Create 2 more segments that can be used to verify MergeTransactionOperation.
    context.containerMetadata.mapStreamSegmentId(badParentName, badParentId);
    UpdateableSegmentMetadata badTransactionMetadata = context.containerMetadata.mapStreamSegmentId(badTransactionName, badTransactionId, badParentId);
    badTransactionMetadata.setLength(0);
    badTransactionMetadata.setStorageLength(0);
    context.storage.create(badTransactionMetadata.getName(), TIMEOUT).join();
    // 1. MergeTransactionOperation
    // 1a.Verify that MergeTransactionOperation cannot be added to the Transaction segment.
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation on the Transaction segment.", () -> transactionAggregator.add(generateSimpleMergeTransaction(transactionMetadata.getId(), context)), ex -> ex instanceof IllegalArgumentException);
    // 1b. Verify that MergeTransactionOperation has the right parent.
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation on the parent for a Transaction that did not have it as a parent.", () -> transactionAggregator.add(generateSimpleMergeTransaction(badTransactionId, context)), ex -> ex instanceof IllegalArgumentException);
    // 2. StreamSegmentSealOperation.
    // 2a. Verify we cannot add a StreamSegmentSealOperation if the segment is not sealed yet.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentSealOperation for a non-sealed segment.", () -> {
        @Cleanup SegmentAggregator badTransactionAggregator = new SegmentAggregator(badTransactionMetadata, context.dataSource, context.storage, DEFAULT_CONFIG, context.timer, executorService());
        badTransactionAggregator.initialize(TIMEOUT).join();
        badTransactionAggregator.add(generateSimpleSeal(badTransactionId, context));
    }, ex -> ex instanceof DataCorruptionException);
    // 2b. Verify that nothing is allowed after Seal (after adding one append to and sealing the Transaction Segment).
    StorageOperation transactionAppend1 = generateAppendAndUpdateMetadata(0, transactionMetadata.getId(), context);
    transactionAggregator.add(transactionAppend1);
    transactionAggregator.add(generateSealAndUpdateMetadata(transactionMetadata.getId(), context));
    AssertExtensions.assertThrows("add() allowed operation after seal.", () -> transactionAggregator.add(generateSimpleAppend(transactionMetadata.getId(), context)), ex -> ex instanceof DataCorruptionException);
    // 3. CachedStreamSegmentAppendOperation.
    final StorageOperation parentAppend1 = generateAppendAndUpdateMetadata(0, SEGMENT_ID, context);
    // 3a. Verify we cannot add StreamSegmentAppendOperations.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentAppendOperation.", () -> {
        // We have the correct offset, but we did not increase the Length.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(parentAppend1.getStreamSegmentId(), parentAppend1.getStreamSegmentOffset(), new byte[(int) parentAppend1.getLength()], null);
        context.segmentAggregator.add(badAppend);
    }, ex -> ex instanceof IllegalArgumentException);
    // Add this one append to the parent (nothing unusual here); we'll use this for the next tests.
    context.segmentAggregator.add(parentAppend1);
    // 3b. Verify we cannot add anything beyond the DurableLogOffset (offset or offset+length).
    AssertExtensions.assertThrows("add() allowed an operation beyond the DurableLogOffset (offset).", () -> {
        // We have the correct offset, but we did not increase the Length.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badAppend));
    }, ex -> ex instanceof DataCorruptionException);
    ((UpdateableSegmentMetadata) context.segmentAggregator.getMetadata()).setLength(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength() + 1);
    AssertExtensions.assertThrows("add() allowed an operation beyond the DurableLogOffset (offset+length).", () -> {
        // We have the correct offset, but we the append exceeds the Length by 1 byte.
        StreamSegmentAppendOperation badAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badAppend));
    }, ex -> ex instanceof DataCorruptionException);
    // 3c. Verify contiguity (offsets - we cannot have gaps in the data).
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too small).", () -> {
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(0);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too large).", () -> {
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength() + 1);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    AssertExtensions.assertThrows("add() allowed an operation with wrong offset (too large, but no pending operations).", () -> {
        @Cleanup SegmentAggregator badTransactionAggregator = new SegmentAggregator(badTransactionMetadata, context.dataSource, context.storage, DEFAULT_CONFIG, context.timer, executorService());
        badTransactionMetadata.setLength(100);
        badTransactionAggregator.initialize(TIMEOUT).join();
        StreamSegmentAppendOperation badOffsetAppend = new StreamSegmentAppendOperation(context.segmentAggregator.getMetadata().getId(), "foo".getBytes(), null);
        badOffsetAppend.setStreamSegmentOffset(1);
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badOffsetAppend));
    }, ex -> ex instanceof DataCorruptionException);
    // 4. Verify Segment Id match.
    AssertExtensions.assertThrows("add() allowed an Append operation with wrong Segment Id.", () -> {
        StreamSegmentAppendOperation badIdAppend = new StreamSegmentAppendOperation(Integer.MAX_VALUE, "foo".getBytes(), null);
        badIdAppend.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(new CachedStreamSegmentAppendOperation(badIdAppend));
    }, ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("add() allowed a StreamSegmentSealOperation with wrong SegmentId.", () -> {
        StreamSegmentSealOperation badIdSeal = new StreamSegmentSealOperation(Integer.MAX_VALUE);
        badIdSeal.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        context.segmentAggregator.add(badIdSeal);
    }, ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("add() allowed a MergeTransactionOperation with wrong SegmentId.", () -> {
        MergeTransactionOperation badIdMerge = new MergeTransactionOperation(Integer.MAX_VALUE, transactionMetadata.getId());
        badIdMerge.setStreamSegmentOffset(parentAppend1.getStreamSegmentOffset() + parentAppend1.getLength());
        badIdMerge.setLength(1);
        context.segmentAggregator.add(badIdMerge);
    }, ex -> ex instanceof IllegalArgumentException);
    // 5. Truncations.
    AssertExtensions.assertThrows("add() allowed a StreamSegmentTruncateOperation with a truncation offset beyond the one in the metadata.", () -> {
        StreamSegmentTruncateOperation op = new StreamSegmentTruncateOperation(SEGMENT_ID, 10);
        op.setSequenceNumber(context.containerMetadata.nextOperationSequenceNumber());
        context.segmentAggregator.add(op);
    }, ex -> ex instanceof DataCorruptionException);
}