Examples with OperationComparer io.pravega.segmentstore.server.logs.operations.OperationComparer used on opensource projects

Example 1 with OperationComparer

use of io.pravega.segmentstore.server.logs.operations.OperationComparer in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, SequencedItemList<Operation> memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).filter(Operation::canSerialize).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    if (successfulOps.size() > 0) {
        // Writing to the memory log is asynchronous and we don't have any callbacks to know when it was written to.
        // We check periodically until the last item has been written.
        await(() -> memoryLog.read(successfulOps.get(successfulOps.size() - 1).getSequenceNumber() - 1, 1).hasNext(), 10);
    }
    Iterator<Operation> memoryLogIterator = memoryLog.read(-1, operations.size() + 1);
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        // Check truncation markers if this is the last Operation to be written.
        LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), dataFrameAddress);
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), dataFrameAddress);
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}

Example 2 with OperationComparer

use of io.pravega.segmentstore.server.logs.operations.OperationComparer in project pravega by pravega.

the class DurableLogTests method performLogOperationChecks.

// endregion
// region Helpers
private void performLogOperationChecks(List<OperationWithCompletion> operations, DurableLog durableLog) {
    // Log Operation based checks
    long lastSeqNo = -1;
    val successfulOperations = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).collect(Collectors.toList());
    // Writing to the DurableLog is done asynchronously, so wait for the last operation to arrive there before reading.
    List<Operation> readOperations = readUpToSequenceNumber(durableLog, successfulOperations.get(successfulOperations.size() - 1).getSequenceNumber());
    val logIterator = readOperations.iterator();
    verifyFirstItemIsMetadataCheckpoint(logIterator);
    OperationComparer comparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOperations) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from DurableLog. Expected: " + expectedOp, logIterator.hasNext());
        // Ok to use assertEquals because we are actually expecting the same object here.
        comparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, logIterator.next());
    }
    Assert.assertFalse(logIterator.hasNext());
}

Example 3 with OperationComparer

use of io.pravega.segmentstore.server.logs.operations.OperationComparer in project pravega by pravega.

the class DurableLogTests method testTailReads.

/**
 * Tests the ability to block reads if the read is at the tail and no more data is available (for now).
 */
@Test
public void testTailReads() throws Exception {
    final int operationCount = 10;
    final long segmentId = 1;
    final String segmentName = Long.toString(segmentId);
    // Setup a DurableLog and start it.
    @Cleanup ContainerSetup setup = new ContainerSetup(executorService());
    @Cleanup DurableLog durableLog = setup.createDurableLog();
    durableLog.startAsync().awaitRunning();
    // Create a segment, which will be used for testing later.
    UpdateableSegmentMetadata segmentMetadata = setup.metadata.mapStreamSegmentId(segmentName, segmentId);
    segmentMetadata.setLength(0);
    segmentMetadata.setStorageLength(0);
    // A MetadataCheckpointOperation gets auto-queued upon the first startup. Get it out of our way for this test.
    val checkpointRead = durableLog.read(1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertTrue("Expected first read operation to be a MetadataCheckpointOperation.", checkpointRead.size() == 1 && checkpointRead.poll() instanceof MetadataCheckpointOperation);
    // Setup a read operation, and make sure it is blocked (since there is no data).
    val readFuture = durableLog.read(operationCount, TIMEOUT);
    Assert.assertFalse("read() returned a completed future when there is no data available", readFuture.isDone());
    // Add one operation and verify that the Read was activated.
    OperationComparer operationComparer = new OperationComparer(true);
    Operation operation = new StreamSegmentAppendOperation(segmentId, new ByteArraySegment("TestData".getBytes()), null);
    durableLog.add(operation, OperationPriority.Normal, TIMEOUT).join();
    // The internal callback happens asynchronously, so wait for this future to complete in a bit.
    val readResult = readFuture.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Verify that we actually have a non-empty read result.
    Assert.assertFalse(readResult.isEmpty());
    // Verify the read result.
    Operation readOp = readResult.poll();
    operationComparer.assertEquals("Unexpected result operation for read.", operation, readOp);
    // Verify that we don't have more than one read result.
    Assert.assertTrue(readResult.isEmpty());
    // Verify that such reads are cancelled when the DurableLog is closed.
    val cancelledRead = durableLog.read(operationCount, TIMEOUT);
    Assert.assertFalse("read() returned a completed future when there is no data available (afterSeqNo = MAX).", cancelledRead.isDone());
    durableLog.stopAsync().awaitTerminated();
    Assert.assertTrue("A tail read was not cancelled when the DurableLog was stopped.", cancelledRead.isCancelled());
}

Example 4 with OperationComparer

use of io.pravega.segmentstore.server.logs.operations.OperationComparer in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, InMemoryLog memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    Iterator<Operation> memoryLogIterator;
    if (successfulOps.isEmpty()) {
        memoryLogIterator = Collections.emptyIterator();
    } else {
        val memoryLogOps = readUpToSequenceNumber(memoryLog, successfulOps.get(successfulOps.size() - 1).getSequenceNumber());
        memoryLogIterator = memoryLogOps.iterator();
    }
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        if (expectedOp instanceof CheckpointOperationBase) {
            // Checkpoint operations are different. While they do serialize their contents, we do not hold on to that
            // since they may be pretty big and serve no purpose after serialization. There are other tests in this suite
            // and in ContainerMetadataUpdateTransactionTests and DurableLogTests that verify we can properly read
            // their contents during recovery.
            val actualEntry = (CheckpointOperationBase) dataFrameRecord.getItem();
            Assert.assertNull("Expected in-memory checkpoint operation to not have contents set.", ((CheckpointOperationBase) expectedOp).getContents());
            Assert.assertNotNull("Expected serialized checkpoint operation to have contents set.", actualEntry.getContents());
            Assert.assertEquals(" Unexpected Sequence Number", expectedOp.getSequenceNumber(), actualEntry.getSequenceNumber());
        } else {
            // All other operations.
            OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        }
        // Check truncation markers if this is the last Operation to be written.
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            AssertExtensions.assertEventuallyEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), () -> truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber()), 100, TIMEOUT.toMillis());
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            AssertExtensions.assertEventuallyEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), () -> truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber()), 100, TIMEOUT.toMillis());
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}