Examples with MergeTransactionOperation io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation used on opensource projects

Example 6 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method populate.

private ArrayList<Operation> populate(MemoryStateUpdater updater, int segmentCount, int operationCountPerType) throws DataCorruptionException {
    ArrayList<Operation> operations = new ArrayList<>();
    long offset = 0;
    for (int i = 0; i < segmentCount; i++) {
        for (int j = 0; j < operationCountPerType; j++) {
            StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name("a").length(i * j).build());
            mapOp.setStreamSegmentId(i);
            operations.add(mapOp);
            StreamSegmentAppendOperation appendOp = new StreamSegmentAppendOperation(i, Integer.toString(i).getBytes(), null);
            appendOp.setStreamSegmentOffset(offset);
            offset += appendOp.getData().length;
            operations.add(appendOp);
            operations.add(new MergeTransactionOperation(i, j));
        }
    }
    for (int i = 0; i < operations.size(); i++) {
        operations.get(i).setSequenceNumber(i);
    }
    updater.process(operations.iterator());
    return operations;
}

Example 7 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method testProcess.

/**
 * Tests the functionality of the process() method.
 */
@Test
public void testProcess() throws Exception {
    int segmentCount = 10;
    int operationCountPerType = 5;
    // Add to MTL + Add to ReadIndex (append; beginMerge).
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    AtomicInteger flushCallbackCallCount = new AtomicInteger();
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, flushCallbackCallCount::incrementAndGet);
    ArrayList<Operation> operations = populate(updater, segmentCount, operationCountPerType);
    // Verify they were properly processed.
    int triggerFutureCount = (int) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).count();
    int addCount = methodInvocations.size() - triggerFutureCount;
    Assert.assertEquals("Unexpected number of items added to ReadIndex.", operations.size() - segmentCount * operationCountPerType, addCount);
    Assert.assertEquals("Unexpected number of calls to the ReadIndex triggerFutureReads method.", 1, triggerFutureCount);
    Assert.assertEquals("Unexpected number of calls to the flushCallback provided in the constructor.", 1, flushCallbackCallCount.get());
    // Verify add calls.
    Iterator<Operation> logIterator = opLog.read(-1, operations.size());
    int currentIndex = -1;
    int currentReadIndex = -1;
    while (logIterator.hasNext()) {
        currentIndex++;
        Operation expected = operations.get(currentIndex);
        Operation actual = logIterator.next();
        if (expected instanceof StorageOperation) {
            currentReadIndex++;
            TestReadIndex.MethodInvocation invokedMethod = methodInvocations.get(currentReadIndex);
            if (expected instanceof StreamSegmentAppendOperation) {
                Assert.assertTrue("StreamSegmentAppendOperation was not added as a CachedStreamSegmentAppendOperation to the Memory Log.", actual instanceof CachedStreamSegmentAppendOperation);
                StreamSegmentAppendOperation appendOp = (StreamSegmentAppendOperation) expected;
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.APPEND, invokedMethod.methodName);
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentId(), invokedMethod.args.get("streamSegmentId"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getData(), invokedMethod.args.get("data"));
            } else if (expected instanceof MergeTransactionOperation) {
                MergeTransactionOperation mergeOp = (MergeTransactionOperation) expected;
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.BEGIN_MERGE, invokedMethod.methodName);
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentId(), invokedMethod.args.get("targetStreamSegmentId"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getTransactionSegmentId(), invokedMethod.args.get("sourceStreamSegmentId"));
            }
        }
    }
    // Verify triggerFutureReads args.
    @SuppressWarnings("unchecked") Collection<Long> triggerSegmentIds = (Collection<Long>) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).findFirst().get().args.get("streamSegmentIds");
    val expectedSegmentIds = operations.stream().filter(op -> op instanceof SegmentOperation).map(op -> ((SegmentOperation) op).getStreamSegmentId()).collect(Collectors.toSet());
    AssertExtensions.assertContainsSameElements("ReadIndex.triggerFutureReads() was called with the wrong set of StreamSegmentIds.", expectedSegmentIds, triggerSegmentIds);
    // Test DataCorruptionException.
    AssertExtensions.assertThrows("MemoryStateUpdater accepted an operation that was out of order.", // This does not have a SequenceNumber set, so it should trigger a DCE.
    () -> updater.process(new MergeTransactionOperation(1, 2)), ex -> ex instanceof DataCorruptionException);
}

Example 8 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class OperationLogTestBase method generateOperations.

// endregion
// region Operation Generation
/**
 * Generates a List of Log Operations that contains the following operations, in the "correct" order.
 * <ol>
 * <li> A set of StreamSegmentAppend Operations (based on the streamSegmentIds arg).
 * <li> A set of StreamSegmentSeal and MergeTransaction Operations (based on the TransactionIds and mergeTransactions arg).
 * <li> A set of StreamSegmentSeal Operations (based on the sealStreamSegments arg).
 * </ol>
 */
List<Operation> generateOperations(Collection<Long> streamSegmentIds, Map<Long, Long> transactionIds, int appendsPerStreamSegment, int metadataCheckpointsEvery, boolean mergeTransactions, boolean sealStreamSegments) {
    List<Operation> result = new ArrayList<>();
    // Add some appends.
    int appendId = 0;
    for (long streamSegmentId : streamSegmentIds) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(streamSegmentId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    for (long transactionId : transactionIds.keySet()) {
        for (int i = 0; i < appendsPerStreamSegment; i++) {
            val attributes = Collections.singletonList(new AttributeUpdate(UUID.randomUUID(), AttributeUpdateType.Replace, i));
            result.add(new StreamSegmentAppendOperation(transactionId, generateAppendData(appendId), attributes));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            appendId++;
        }
    }
    addProbe(result);
    // Merge Transactions.
    if (mergeTransactions) {
        // Key = TransactionId, Value = Parent Id.
        transactionIds.entrySet().forEach(mapping -> {
            result.add(new StreamSegmentSealOperation(mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
            result.add(new MergeTransactionOperation(mapping.getValue(), mapping.getKey()));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    // Seal the StreamSegments.
    if (sealStreamSegments) {
        streamSegmentIds.forEach(streamSegmentId -> {
            result.add(new StreamSegmentSealOperation(streamSegmentId));
            addCheckpointIfNeeded(result, metadataCheckpointsEvery);
        });
        addProbe(result);
    }
    return result;
}

Example 9 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAcceptMergeTransaction.

/**
 * Tests the accept method with MergeTransactionOperations.
 */
@Test
public void testAcceptMergeTransaction() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    MergeTransactionOperation mergeOp = createMerge();
    // When no pre-process has happened
    AssertExtensions.assertThrows("Unexpected behavior from acceptOperation() when no pre-processing was made.", () -> txn.acceptOperation(mergeOp), ex -> ex instanceof MetadataUpdateException);
    Assert.assertEquals("acceptOperation updated the transaction even if it threw an exception (parent segment).", SEGMENT_LENGTH, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata (parent segment).", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // This would naturally happen in case of a failure, so we need to simulate this here too.
    txn.clear();
    // When all is good.
    txn.preProcessOperation(mergeOp);
    txn.acceptOperation(mergeOp);
    Assert.assertTrue("acceptOperation did not update the transaction(Transaction).", txn.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("acceptOperation updated the metadata (Transaction).", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertEquals("acceptOperation did not update the transaction.", SEGMENT_LENGTH + SEALED_TRANSACTION_LENGTH, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("acceptOperation updated the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
}

Example 10 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessMergeTransaction.

// endregion
// region MergeTransactionOperation
/**
 * Tests the preProcess method with MergeTransactionOperations.
 * Scenarios:
 * * Recovery/non-recovery mode
 * * Target StreamSegment is sealed
 * * Target StreamSegment is a Transaction
 * * Transaction StreamSegment is already merged
 * * Transaction StreamSegment is not sealed
 */
@Test
public void testPreProcessMergeTransaction() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // When everything is OK (recovery mode).
    MergeTransactionOperation recoveryMergeOp = createMerge();
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    AssertExtensions.assertThrows("preProcess(Merge) handled an operation with no Transaction StreamSegment Length set.", () -> txn1.preProcessOperation(createMerge()), ex -> ex instanceof MetadataUpdateException);
    // In recovery mode, the updater does not set the length; it just validates that it has one.
    recoveryMergeOp.setLength(metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).getLength());
    txn1.preProcessOperation(recoveryMergeOp);
    AssertExtensions.assertLessThan("Unexpected Target StreamSegmentOffset after call to preProcess in recovery mode.", 0, recoveryMergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(recoveryMergeOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When everything is OK (non-recovery mode).
    MergeTransactionOperation mergeOp = createMerge();
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mergeOp);
    Assert.assertEquals("Unexpected Transaction StreamSegmentLength after call to preProcess in non-recovery mode.", SEALED_TRANSACTION_LENGTH, mergeOp.getLength());
    Assert.assertEquals("Unexpected Target StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, mergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(mergeOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in non-recovery mode.", txn2.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in non-recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When Target StreamSegment is sealed.
    StreamSegmentSealOperation sealTargetOp = createSeal();
    txn2.preProcessOperation(sealTargetOp);
    txn2.acceptOperation(sealTargetOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is sealed.", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentSealedException);
    // Rollback the seal
    txn2.clear();
    // When Target StreamSegment is a Transaction.
    MergeTransactionOperation mergeToTransactionOp = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEALED_TRANSACTION_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is a Transaction.", () -> txn2.preProcessOperation(mergeToTransactionOp), ex -> ex instanceof MetadataUpdateException);
    // When Transaction is not sealed.
    MergeTransactionOperation mergeNonSealed = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEGMENT_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is not sealed.", () -> txn2.preProcessOperation(mergeNonSealed), ex -> ex instanceof StreamSegmentNotSealedException);
    // When Transaction is already merged.
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in transaction).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
    txn2.commit(metadata);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in metadata).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
}