Examples with StreamSegmentSealOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation used on opensource projects

Example 16 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method acceptOperation.

/**
 * Accepts the given Operation. The Operation's effects are reflected in the pending transaction.
 * This method has no effect on Metadata Operations.
 * See OperationMetadataUpdater.acceptOperation for more details on behavior.
 *
 * @param operation The operation to accept.
 * @throws MetadataUpdateException If the given operation was rejected given the current state of the metadata.
 * @throws NullPointerException    If the operation is null.
 */
void acceptOperation(Operation operation) throws MetadataUpdateException {
    checkNotSealed();
    if (operation instanceof SegmentOperation) {
        val segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        segmentMetadata.setLastUsed(operation.getSequenceNumber());
        if (operation instanceof StreamSegmentAppendOperation) {
            segmentMetadata.acceptOperation((StreamSegmentAppendOperation) operation);
        } else if (operation instanceof StreamSegmentSealOperation) {
            segmentMetadata.acceptOperation((StreamSegmentSealOperation) operation);
        } else if (operation instanceof MergeSegmentOperation) {
            MergeSegmentOperation mto = (MergeSegmentOperation) operation;
            SegmentMetadataUpdateTransaction sourceMetadata = getSegmentUpdateTransaction(mto.getSourceSegmentId());
            sourceMetadata.acceptAsSourceSegment(mto);
            sourceMetadata.setLastUsed(operation.getSequenceNumber());
            segmentMetadata.acceptAsTargetSegment(mto, sourceMetadata);
        } else if (operation instanceof UpdateAttributesOperation) {
            segmentMetadata.acceptOperation((UpdateAttributesOperation) operation);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            segmentMetadata.acceptOperation((StreamSegmentTruncateOperation) operation);
        } else if (operation instanceof DeleteSegmentOperation) {
            segmentMetadata.acceptOperation((DeleteSegmentOperation) operation);
        }
    }
    if (operation instanceof CheckpointOperationBase) {
        if (operation instanceof MetadataCheckpointOperation) {
            // A MetadataCheckpointOperation represents a valid truncation point. Record it as such.
            this.newTruncationPoints.add(operation.getSequenceNumber());
        }
        // Checkpoint operation has been serialized and we no longer need its contents. Clear it and release any
        // memory it used.
        ((CheckpointOperationBase) operation).clearContents();
    } else if (operation instanceof StreamSegmentMapOperation) {
        acceptMetadataOperation((StreamSegmentMapOperation) operation);
    }
}

Example 17 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class DurableLogTests method testRecoveryWithIncrementalCheckpoints.

/**
 * Tests the DurableLog recovery process when there are multiple {@link MetadataCheckpointOperation}s added, with each
 * such checkpoint including information about evicted segments or segments which had their storage state modified.
 */
@Test
public void testRecoveryWithIncrementalCheckpoints() throws Exception {
    final int streamSegmentCount = 50;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    // First DurableLog. We use this for generating data.
    val metadata1 = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
    @Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
    List<Long> deletedIds;
    Set<Long> evictIds;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create some segments.
        val segmentIds = new ArrayList<>(createStreamSegmentsWithOperations(streamSegmentCount, durableLog));
        deletedIds = segmentIds.subList(0, 5);
        val mergedFromIds = segmentIds.subList(5, 10);
        // Must be same length as mergeFrom
        val mergedToIds = segmentIds.subList(10, 15);
        evictIds = new HashSet<>(segmentIds.subList(15, 20));
        val changeStorageStateIds = segmentIds.subList(20, segmentIds.size() - 5);
        // Append something to each segment.
        for (val segmentId : segmentIds) {
            if (!evictIds.contains(segmentId)) {
                durableLog.add(new StreamSegmentAppendOperation(segmentId, generateAppendData((int) (long) segmentId), null), OperationPriority.Normal, TIMEOUT).join();
            }
        }
        // Checkpoint 1.
        durableLog.checkpoint(TIMEOUT).join();
        // Delete some segments.
        for (val segmentId : deletedIds) {
            durableLog.add(new DeleteSegmentOperation(segmentId), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 2.
        durableLog.checkpoint(TIMEOUT).join();
        // Merge some segments.
        for (int i = 0; i < mergedFromIds.size(); i++) {
            durableLog.add(new StreamSegmentSealOperation(mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
            durableLog.add(new MergeSegmentOperation(mergedToIds.get(i), mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 3.
        durableLog.checkpoint(TIMEOUT).join();
        // Evict some segments.
        val evictableContainerMetadata = (EvictableMetadata) metadata1;
        metadata1.removeTruncationMarkers(metadata1.getOperationSequenceNumber());
        val toEvict = evictableContainerMetadata.getEvictionCandidates(Integer.MAX_VALUE, segmentIds.size()).stream().filter(m -> evictIds.contains(m.getId())).collect(Collectors.toList());
        val evicted = evictableContainerMetadata.cleanup(toEvict, Integer.MAX_VALUE);
        AssertExtensions.assertContainsSameElements("", evictIds, evicted.stream().map(SegmentMetadata::getId).collect(Collectors.toList()));
        // Checkpoint 4.
        durableLog.checkpoint(TIMEOUT).join();
        // Update storage state for some segments.
        for (val segmentId : changeStorageStateIds) {
            val sm = metadata1.getStreamSegmentMetadata(segmentId);
            if (segmentId % 3 == 0) {
                sm.setStorageLength(sm.getLength());
            }
            if (segmentId % 4 == 0) {
                sm.markSealed();
                sm.markSealedInStorage();
            }
            if (segmentId % 5 == 0) {
                sm.markDeleted();
                sm.markDeletedInStorage();
            }
        }
        // Checkpoint 5.
        durableLog.checkpoint(TIMEOUT).join();
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Second DurableLog. We use this for recovery.
    val metadata2 = new MetadataBuilder(CONTAINER_ID).build();
    try (ContainerReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Validate metadata matches.
        val expectedSegmentIds = metadata1.getAllStreamSegmentIds();
        val actualSegmentIds = metadata2.getAllStreamSegmentIds();
        AssertExtensions.assertContainsSameElements("Unexpected set of recovered segments. Only Active segments expected to have been recovered.", expectedSegmentIds, actualSegmentIds);
        val expectedSegments = expectedSegmentIds.stream().sorted().map(metadata1::getStreamSegmentMetadata).collect(Collectors.toList());
        val actualSegments = actualSegmentIds.stream().sorted().map(metadata2::getStreamSegmentMetadata).collect(Collectors.toList());
        for (int i = 0; i < expectedSegments.size(); i++) {
            val e = expectedSegments.get(i);
            val a = actualSegments.get(i);
            SegmentMetadataComparer.assertEquals("Recovered segment metadata mismatch", e, a);
        }
        // Validate read index is as it should. Here, we can only check if the read indices for evicted segments are
        // no longer loaded; we do more thorough checks in the ContainerReadIndexTests suite.
        Streams.concat(evictIds.stream(), deletedIds.stream()).forEach(segmentId -> Assert.assertNull("Not expecting a read index for an evicted or deleted segment.", readIndex.getIndex(segmentId)));
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}

Example 18 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class OperationMetadataUpdaterTests method sealSegment.

private void sealSegment(long segmentId, OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception {
    val op = new StreamSegmentSealOperation(segmentId);
    process(op, updater);
    if (referenceMetadata != null) {
        referenceMetadata.getStreamSegmentMetadata(segmentId).markSealed();
    }
}

Example 19 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class DurableDataLogRepairCommand method createUserDefinedOperation.

/**
 * Guides the user to generate a new {@link Operation} that will eventually modify the Original Log.
 *
 * @return New {@link Operation} to be added in the Original Log.
 */
@VisibleForTesting
Operation createUserDefinedOperation() {
    Operation result;
    final String operations = "[DeleteSegmentOperation|MergeSegmentOperation|MetadataCheckpointOperation|" + "StorageMetadataCheckpointOperation|StreamSegmentAppendOperation|StreamSegmentMapOperation|" + "StreamSegmentSealOperation|StreamSegmentTruncateOperation|UpdateAttributesOperation]";
    switch(getStringUserInput("Type one of the following Operations to instantiate: " + operations)) {
        case "DeleteSegmentOperation":
            long segmentId = getLongUserInput("Input Segment Id for DeleteSegmentOperation:");
            result = new DeleteSegmentOperation(segmentId);
            long offset = getLongUserInput("Input Segment Offset for DeleteSegmentOperation:");
            ((DeleteSegmentOperation) result).setStreamSegmentOffset(offset);
            break;
        case "MergeSegmentOperation":
            long targetSegmentId = getLongUserInput("Input Target Segment Id for MergeSegmentOperation:");
            long sourceSegmentId = getLongUserInput("Input Source Segment Id for MergeSegmentOperation:");
            result = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, createAttributeUpdateCollection());
            offset = getLongUserInput("Input Segment Offset for MergeSegmentOperation:");
            ((MergeSegmentOperation) result).setStreamSegmentOffset(offset);
            break;
        case "MetadataCheckpointOperation":
            result = new MetadataCheckpointOperation();
            ((MetadataCheckpointOperation) result).setContents(createOperationContents());
            break;
        case "StorageMetadataCheckpointOperation":
            result = new StorageMetadataCheckpointOperation();
            ((StorageMetadataCheckpointOperation) result).setContents(createOperationContents());
            break;
        case "StreamSegmentAppendOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentAppendOperation:");
            offset = getLongUserInput("Input Segment Offset for StreamSegmentAppendOperation:");
            result = new StreamSegmentAppendOperation(segmentId, offset, createOperationContents(), createAttributeUpdateCollection());
            break;
        case "StreamSegmentMapOperation":
            result = new StreamSegmentMapOperation(createSegmentProperties());
            break;
        case "StreamSegmentSealOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentSealOperation:");
            result = new StreamSegmentSealOperation(segmentId);
            offset = getLongUserInput("Input Segment Offset for StreamSegmentSealOperation:");
            ((StreamSegmentSealOperation) result).setStreamSegmentOffset(offset);
            break;
        case "StreamSegmentTruncateOperation":
            segmentId = getLongUserInput("Input Segment Id for StreamSegmentTruncateOperation:");
            offset = getLongUserInput("Input Offset for StreamSegmentTruncateOperation:");
            result = new StreamSegmentTruncateOperation(segmentId, offset);
            break;
        case "UpdateAttributesOperation":
            segmentId = getLongUserInput("Input Segment Id for UpdateAttributesOperation:");
            result = new UpdateAttributesOperation(segmentId, createAttributeUpdateCollection());
            break;
        default:
            output("Invalid operation, please select one of " + operations);
            throw new UnsupportedOperationException();
    }
    return result;
}

Example 20 with StreamSegmentSealOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessMergeSegment.

// endregion
// region MergeSegmentOperation
/**
 * Tests the preProcess method with MergeTransactionOperations.
 * Scenarios:
 * * Recovery/non-recovery mode
 * * Target StreamSegment is sealed
 * * Target StreamSegment is a Transaction
 * * Source StreamSegment is already merged
 * * Source StreamSegment is not sealed
 */
@Test
public void testPreProcessMergeSegment() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // When everything is OK (recovery mode).
    MergeSegmentOperation recoveryMergeOp = createMerge();
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    AssertExtensions.assertThrows("preProcess(Merge) handled an operation with no Transaction StreamSegment Length set.", () -> txn1.preProcessOperation(createMerge()), ex -> ex instanceof MetadataUpdateException);
    // In recovery mode, the updater does not set the length; it just validates that it has one.
    recoveryMergeOp.setLength(metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).getLength());
    txn1.preProcessOperation(recoveryMergeOp);
    AssertExtensions.assertLessThan("Unexpected Target StreamSegmentOffset after call to preProcess in recovery mode.", 0, recoveryMergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(recoveryMergeOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    // When everything is OK (non-recovery mode).
    MergeSegmentOperation mergeOp = createMerge();
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mergeOp);
    Assert.assertEquals("Unexpected Transaction StreamSegmentLength after call to preProcess in non-recovery mode.", SEALED_SOURCE_LENGTH, mergeOp.getLength());
    Assert.assertEquals("Unexpected Target StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, mergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(mergeOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in non-recovery mode.", txn2.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in non-recovery mode.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    // When Target StreamSegment is sealed.
    StreamSegmentSealOperation sealTargetOp = createSeal();
    txn2.preProcessOperation(sealTargetOp);
    txn2.acceptOperation(sealTargetOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is sealed.", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentSealedException);
    // Rollback the seal
    txn2.clear();
    // When Transaction is not sealed.
    MergeSegmentOperation mergeNonSealed = new MergeSegmentOperation(NOTSEALED_SOURCE_ID, SEGMENT_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is not sealed.", () -> txn2.preProcessOperation(mergeNonSealed), ex -> ex instanceof StreamSegmentNotSealedException);
    // When Transaction is already merged.
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in transaction).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
    txn2.commit(metadata);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in metadata).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
}