Examples with TruncationMarkerRepository io.pravega.segmentstore.server.TruncationMarkerRepository used on opensource projects

Example 1 with TruncationMarkerRepository

use of io.pravega.segmentstore.server.TruncationMarkerRepository in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, SequencedItemList<Operation> memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).filter(Operation::canSerialize).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    if (successfulOps.size() > 0) {
        // Writing to the memory log is asynchronous and we don't have any callbacks to know when it was written to.
        // We check periodically until the last item has been written.
        await(() -> memoryLog.read(successfulOps.get(successfulOps.size() - 1).getSequenceNumber() - 1, 1).hasNext(), 10);
    }
    Iterator<Operation> memoryLogIterator = memoryLog.read(-1, operations.size() + 1);
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        // Check truncation markers if this is the last Operation to be written.
        LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), dataFrameAddress);
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), dataFrameAddress);
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}