Examples with CheckpointOperationBase io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase used on opensource projects

Example 1 with CheckpointOperationBase

use of io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase in project pravega by pravega.

the class RecoveryProcessor method recoverOperation.

protected void recoverOperation(DataFrameRecord<Operation> dataFrameRecord, OperationMetadataUpdater metadataUpdater) throws ServiceHaltException {
    // Update Metadata Sequence Number.
    Operation operation = dataFrameRecord.getItem();
    metadataUpdater.setOperationSequenceNumber(operation.getSequenceNumber());
    // Update the metadata with the information from the Operation.
    try {
        log.debug("{} Recovering {}.", this.traceObjectId, operation);
        metadataUpdater.preProcessOperation(operation);
        metadataUpdater.acceptOperation(operation);
    } catch (StreamSegmentException | ContainerException ex) {
        // Metadata update failures should not happen during recovery.
        throw new DataCorruptionException(String.format("Unable to update metadata for Log Operation '%s'.", operation), ex);
    }
    // Update in-memory structures.
    this.stateUpdater.process(operation);
    // Perform necessary read index cleanups if possible.
    if (operation instanceof CheckpointOperationBase) {
        this.stateUpdater.cleanupReadIndex();
    }
}

Example 2 with CheckpointOperationBase

use of io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, InMemoryLog memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    Iterator<Operation> memoryLogIterator;
    if (successfulOps.isEmpty()) {
        memoryLogIterator = Collections.emptyIterator();
    } else {
        val memoryLogOps = readUpToSequenceNumber(memoryLog, successfulOps.get(successfulOps.size() - 1).getSequenceNumber());
        memoryLogIterator = memoryLogOps.iterator();
    }
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        if (expectedOp instanceof CheckpointOperationBase) {
            // Checkpoint operations are different. While they do serialize their contents, we do not hold on to that
            // since they may be pretty big and serve no purpose after serialization. There are other tests in this suite
            // and in ContainerMetadataUpdateTransactionTests and DurableLogTests that verify we can properly read
            // their contents during recovery.
            val actualEntry = (CheckpointOperationBase) dataFrameRecord.getItem();
            Assert.assertNull("Expected in-memory checkpoint operation to not have contents set.", ((CheckpointOperationBase) expectedOp).getContents());
            Assert.assertNotNull("Expected serialized checkpoint operation to have contents set.", actualEntry.getContents());
            Assert.assertEquals(" Unexpected Sequence Number", expectedOp.getSequenceNumber(), actualEntry.getSequenceNumber());
        } else {
            // All other operations.
            OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        }
        // Check truncation markers if this is the last Operation to be written.
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            AssertExtensions.assertEventuallyEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), () -> truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber()), 100, TIMEOUT.toMillis());
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            AssertExtensions.assertEventuallyEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), () -> truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber()), 100, TIMEOUT.toMillis());
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}

Example 3 with CheckpointOperationBase

use of io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method acceptOperation.

/**
 * Accepts the given Operation. The Operation's effects are reflected in the pending transaction.
 * This method has no effect on Metadata Operations.
 * See OperationMetadataUpdater.acceptOperation for more details on behavior.
 *
 * @param operation The operation to accept.
 * @throws MetadataUpdateException If the given operation was rejected given the current state of the metadata.
 * @throws NullPointerException    If the operation is null.
 */
void acceptOperation(Operation operation) throws MetadataUpdateException {
    checkNotSealed();
    if (operation instanceof SegmentOperation) {
        val segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        segmentMetadata.setLastUsed(operation.getSequenceNumber());
        if (operation instanceof StreamSegmentAppendOperation) {
            segmentMetadata.acceptOperation((StreamSegmentAppendOperation) operation);
        } else if (operation instanceof StreamSegmentSealOperation) {
            segmentMetadata.acceptOperation((StreamSegmentSealOperation) operation);
        } else if (operation instanceof MergeSegmentOperation) {
            MergeSegmentOperation mto = (MergeSegmentOperation) operation;
            SegmentMetadataUpdateTransaction sourceMetadata = getSegmentUpdateTransaction(mto.getSourceSegmentId());
            sourceMetadata.acceptAsSourceSegment(mto);
            sourceMetadata.setLastUsed(operation.getSequenceNumber());
            segmentMetadata.acceptAsTargetSegment(mto, sourceMetadata);
        } else if (operation instanceof UpdateAttributesOperation) {
            segmentMetadata.acceptOperation((UpdateAttributesOperation) operation);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            segmentMetadata.acceptOperation((StreamSegmentTruncateOperation) operation);
        } else if (operation instanceof DeleteSegmentOperation) {
            segmentMetadata.acceptOperation((DeleteSegmentOperation) operation);
        }
    }
    if (operation instanceof CheckpointOperationBase) {
        if (operation instanceof MetadataCheckpointOperation) {
            // A MetadataCheckpointOperation represents a valid truncation point. Record it as such.
            this.newTruncationPoints.add(operation.getSequenceNumber());
        }
        // Checkpoint operation has been serialized and we no longer need its contents. Clear it and release any
        // memory it used.
        ((CheckpointOperationBase) operation).clearContents();
    } else if (operation instanceof StreamSegmentMapOperation) {
        acceptMetadataOperation((StreamSegmentMapOperation) operation);
    }
}

Example 4 with CheckpointOperationBase

use of io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase in project pravega by pravega.

the class DurableLogTests method assertRecoveredOperationsMatch.

private void assertRecoveredOperationsMatch(List<Operation> expected, List<Operation> actual) {
    Assert.assertEquals("Recovered operations do not match original ones. Collections differ in size.", expected.size(), actual.size());
    for (int i = 0; i < expected.size(); i++) {
        Operation expectedItem = expected.get(i);
        Operation actualItem = actual.get(i);
        if (expectedItem instanceof CheckpointOperationBase) {
            Assert.assertNull("Recovered Checkpoint Operation did not have contents cleared up.", ((CheckpointOperationBase) actualItem).getContents());
            Assert.assertEquals(" Unexpected Sequence Number", expectedItem.getSequenceNumber(), actualItem.getSequenceNumber());
        } else {
            OperationComparer.DEFAULT.assertEquals(String.format("Recovered operations do not match original ones. Elements at index %d differ. Expected '%s', found '%s'.", i, expectedItem, actualItem), expectedItem, actualItem);
        }
    }
}

Example 5 with CheckpointOperationBase

use of io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase in project pravega by pravega.

the class OperationProcessorTests method performFailureRecoveryChecks.

private void performFailureRecoveryChecks(List<OperationWithCompletion> operations, TestDurableDataLog dataLog) throws Exception {
    boolean encounteredFailure = false;
    val successfulOps = new ArrayList<Operation>();
    for (val oc : operations) {
        encounteredFailure |= oc.completion.isCompletedExceptionally();
        if (encounteredFailure) {
            Assert.assertTrue("Found a non-errorred operation after an errorred operation.", oc.completion.isCompletedExceptionally());
        } else {
            Assert.assertTrue(oc.completion.isDone());
            successfulOps.add(oc.operation);
        }
    }
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        if (expectedOp instanceof CheckpointOperationBase) {
            // Checkpoint operations are different. While they do serialize their contents, we do not hold on to that
            // since they may be pretty big and serve no purpose after serialization. There are other tests in this suite
            // and in ContainerMetadataUpdateTransactionTests and DurableLogTests that verify we can properly read
            // their contents during recovery.
            val actualEntry = (CheckpointOperationBase) dataFrameRecord.getItem();
            Assert.assertNull("Expected in-memory checkpoint operation to not have contents set.", ((CheckpointOperationBase) expectedOp).getContents());
            Assert.assertNotNull("Expected serialized checkpoint operation to have contents set.", actualEntry.getContents());
            Assert.assertEquals(" Unexpected Sequence Number", expectedOp.getSequenceNumber(), actualEntry.getSequenceNumber());
        } else {
            // All other operations.
            OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        }
    }
}