Examples with StreamSegmentTruncateOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation used on opensource projects

Example 1 with StreamSegmentTruncateOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationUserInput.

@Test
public void testRepairLogEditOperationUserInput() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Case 1: Input a Delete Edit Operation with wrong initial/final ids. Then retry with correct ids.
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").when(command).getStringUserInput(Mockito.any());
    Assert.assertEquals(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 2, null)), command.getDurableLogEditsFromUser());
    // Case 2: Input an Add Edit Operation with a wrong operation type. Then retry with correct operation type.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("wrong").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    DeleteSegmentOperation deleteOperationAdded = new DeleteSegmentOperation(1);
    List<DurableDataLogRepairCommand.LogEditOperation> editOps = new ArrayList<>();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 3: Create rest of operation types without payload (MergeSegmentOperation, StreamSegmentMapOperation, StreamSegmentTruncateOperation, UpdateAttributesOperation)
    long timestamp = System.currentTimeMillis();
    UUID uuid = UUID.randomUUID();
    editOps.clear();
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(2L).doReturn(1L).doReturn(2L).doReturn(123L).doReturn(2L).doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).doReturn(3L).doReturn(3L).doReturn(4L).doReturn(4L).doReturn(3L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("MergeSegmentOperation").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentMapOperation").doReturn("test").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentTruncateOperation").doReturn("add").doReturn("UpdateAttributesOperation").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn((int) AttributeUpdateType.Replace.getTypeId()).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(AttributeId.fromUUID(uuid), AttributeUpdateType.Replace, 1, 2));
    MergeSegmentOperation mergeSegmentOperation = new MergeSegmentOperation(1, 2, attributeUpdates);
    mergeSegmentOperation.setStreamSegmentOffset(123);
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, mergeSegmentOperation));
    Map<AttributeId, Long> attributes = new HashMap<>();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 2, 2, new StreamSegmentMapOperation(segmentProperties)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 3, 3, new StreamSegmentTruncateOperation(3, 3)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 4, 4, new UpdateAttributesOperation(4, attributeUpdates)));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 4: Add wrong inputs.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doThrow(NumberFormatException.class).doThrow(NullPointerException.class).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("wrong").doReturn("replace").doReturn("replace").when(command).getStringUserInput(Mockito.any());
    command.getDurableLogEditsFromUser();
}

Example 2 with StreamSegmentTruncateOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation in project pravega by pravega.

the class SegmentAggregatorTests method testTruncateAlreadySealedSegment.

/**
 * Tests the SegmentAggregator's behavior when an already Sealed Segment is opened and truncated.
 */
@Test
public void testTruncateAlreadySealedSegment() throws Exception {
    // Pre-create the segment, write some data, and then seal it.
    val rnd = new Random(0);
    byte[] storageData = new byte[100];
    rnd.nextBytes(storageData);
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG);
    // Create a segment, add some data, and seal it in storage.
    context.storage.create(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join();
    context.storage.openWrite(context.segmentAggregator.getMetadata().getName()).thenCompose(h -> context.storage.write(h, 0, new ByteArrayInputStream(storageData), storageData.length, TIMEOUT).thenCompose(v -> context.storage.seal(h, TIMEOUT))).join();
    val sm = context.containerMetadata.getStreamSegmentMetadata(context.segmentAggregator.getMetadata().getId());
    sm.setLength(storageData.length);
    sm.setStorageLength(storageData.length);
    sm.markSealed();
    sm.markSealedInStorage();
    // Initialize the SegmentAggregator.
    context.segmentAggregator.initialize(TIMEOUT).join();
    // Generate and add a Seal Operation.
    StorageOperation truncateOp = generateTruncateAndUpdateMetadata(SEGMENT_ID, context);
    context.segmentAggregator.add(truncateOp);
    Assert.assertTrue("Unexpected value returned by mustFlush() after adding StreamSegmentTruncateOperation.", context.segmentAggregator.mustFlush());
    // Call flush and verify that the entire Aggregator got flushed and the Truncate got persisted to Storage.
    context.segmentAggregator.flush(TIMEOUT).join();
    // Verify data.
    SegmentProperties storageInfo = context.storage.getStreamSegmentInfo(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join();
    Assert.assertEquals("Unexpected number of bytes in Storage.", storageData.length, storageInfo.getLength());
    Assert.assertEquals("Unexpected truncation offset in Storage.", truncateOp.getStreamSegmentOffset(), context.storage.getTruncationOffset(context.segmentAggregator.getMetadata().getName()));
}

Example 3 with StreamSegmentTruncateOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation in project pravega by pravega.

the class AttributeAggregatorTests method testAdd.

/**
 * Tests the {@link AttributeAggregator#add} method with valid and invalid operations.
 */
@Test
public void testAdd() throws Exception {
    final AttributeId extendedId = EXTENDED_ATTRIBUTE_IDS.get(0);
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG);
    // We want to make sure we do not prematurely acknowledge anything.
    context.dataSource.setCompleteMergeCallback((target, source) -> Assert.fail("Not expecting any merger callbacks yet."));
    // Add non-Attributes.
    context.aggregator.add(new StreamSegmentTruncateOperation(SEGMENT_ID, 1234));
    // Add some attributes
    context.aggregator.add(new StreamSegmentAppendOperation(SEGMENT_ID, new ByteArraySegment(new byte[123]), AttributeUpdateCollection.from(createAttributeUpdate(extendedId, 1))));
    context.aggregator.add(new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(createAttributeUpdate(extendedId, 2))));
    Assert.assertFalse("Unexpected value from mustFlush().", context.aggregator.mustFlush());
    // Seal using operation.
    context.aggregator.add(new StreamSegmentSealOperation(SEGMENT_ID));
    Assert.assertTrue("Unexpected value from mustFlush() after sealing.", context.aggregator.mustFlush());
    // Verify further adds fail.
    AssertExtensions.assertThrows("No operations should be allowed after sealing.", () -> context.aggregator.add(new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(createAttributeUpdate(extendedId, 3)))), ex -> ex instanceof DataCorruptionException);
    // Delete and verify nothing else changes.
    context.segmentMetadata.markDeleted();
    context.aggregator.add(new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(createAttributeUpdate(extendedId, 4))));
    Assert.assertFalse("Unexpected value from mustFlush() after deleting.", context.aggregator.mustFlush());
}

Example 4 with StreamSegmentTruncateOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation in project pravega by pravega.

the class StreamSegmentContainer method truncateStreamSegment.

@Override
public CompletableFuture<Void> truncateStreamSegment(String streamSegmentName, long offset, Duration timeout) {
    ensureRunning();
    logRequest("truncateStreamSegment", streamSegmentName);
    this.metrics.truncate();
    TimeoutTimer timer = new TimeoutTimer(timeout);
    return this.metadataStore.getOrAssignSegmentId(streamSegmentName, timer.getRemaining(), streamSegmentId -> {
        StreamSegmentTruncateOperation op = new StreamSegmentTruncateOperation(streamSegmentId, offset);
        return addOperation(op, timeout);
    });
}

Example 5 with StreamSegmentTruncateOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method preProcessOperation.

/**
 * Pre-processes the given Operation. See OperationMetadataUpdater.preProcessOperation for more details on behavior.
 *
 * @param operation The operation to pre-process.
 * @throws ContainerException     If the given operation was rejected given the current state of the container metadata.
 * @throws StreamSegmentException If the given operation was incompatible with the current state of the Segment.
 *                                For example: StreamSegmentNotExistsException, StreamSegmentSealedException or
 *                                StreamSegmentMergedException.
 */
void preProcessOperation(Operation operation) throws ContainerException, StreamSegmentException {
    checkNotSealed();
    if (operation instanceof SegmentOperation) {
        val segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        if (segmentMetadata.isDeleted()) {
            throw new StreamSegmentNotExistsException(segmentMetadata.getName());
        }
        if (operation instanceof StreamSegmentAppendOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentAppendOperation) operation);
        } else if (operation instanceof StreamSegmentSealOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentSealOperation) operation);
        } else if (operation instanceof MergeSegmentOperation) {
            MergeSegmentOperation mbe = (MergeSegmentOperation) operation;
            SegmentMetadataUpdateTransaction sourceMetadata = getSegmentUpdateTransaction(mbe.getSourceSegmentId());
            sourceMetadata.preProcessAsSourceSegment(mbe);
            segmentMetadata.preProcessAsTargetSegment(mbe, sourceMetadata);
        } else if (operation instanceof UpdateAttributesOperation) {
            segmentMetadata.preProcessOperation((UpdateAttributesOperation) operation);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentTruncateOperation) operation);
        } else if (operation instanceof DeleteSegmentOperation) {
            segmentMetadata.preProcessOperation((DeleteSegmentOperation) operation);
        }
    }
    if (operation instanceof MetadataCheckpointOperation) {
        // MetadataCheckpointOperations do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((MetadataCheckpointOperation) operation);
    } else if (operation instanceof StorageMetadataCheckpointOperation) {
        // StorageMetadataCheckpointOperation do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((StorageMetadataCheckpointOperation) operation);
    } else if (operation instanceof StreamSegmentMapOperation) {
        preProcessMetadataOperation((StreamSegmentMapOperation) operation);
    }
}