Examples with DeleteSegmentOperation io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation used on opensource projects

Example 1 with DeleteSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationCorrectness.

@Test
public void testRepairLogEditOperationCorrectness() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = new DurableDataLogRepairCommand(args);
    // Check when no commands are available.
    command.checkDurableLogEdits(new ArrayList<>());
    // Check adding Edit Operations with sequence numbers lower than 0.
    AssertExtensions.assertThrows("Edit Operations should have initial sequence ids > 0.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, -1, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 0, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    // A Delete Edit Operation should have an initial sequence number lower than the final one.
    AssertExtensions.assertThrows("Edit Operations should have initial sequence ids > 0.", () -> command.checkDurableLogEdits(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 2, 2, null))), ex -> ex instanceof IllegalStateException);
    // Add one Add Edit and one Replace Edit on the same sequence number. This is expected to fail.
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 10, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    // We can have multiple Add Edit Operations on the same sequence number.
    command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0))));
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 10, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 10, null), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 5, 20, null))), ex -> ex instanceof IllegalStateException);
}

Example 2 with DeleteSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationUserInput.

@Test
public void testRepairLogEditOperationUserInput() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Case 1: Input a Delete Edit Operation with wrong initial/final ids. Then retry with correct ids.
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").when(command).getStringUserInput(Mockito.any());
    Assert.assertEquals(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 2, null)), command.getDurableLogEditsFromUser());
    // Case 2: Input an Add Edit Operation with a wrong operation type. Then retry with correct operation type.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("wrong").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    DeleteSegmentOperation deleteOperationAdded = new DeleteSegmentOperation(1);
    List<DurableDataLogRepairCommand.LogEditOperation> editOps = new ArrayList<>();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 3: Create rest of operation types without payload (MergeSegmentOperation, StreamSegmentMapOperation, StreamSegmentTruncateOperation, UpdateAttributesOperation)
    long timestamp = System.currentTimeMillis();
    UUID uuid = UUID.randomUUID();
    editOps.clear();
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(2L).doReturn(1L).doReturn(2L).doReturn(123L).doReturn(2L).doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).doReturn(3L).doReturn(3L).doReturn(4L).doReturn(4L).doReturn(3L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("MergeSegmentOperation").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentMapOperation").doReturn("test").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentTruncateOperation").doReturn("add").doReturn("UpdateAttributesOperation").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn((int) AttributeUpdateType.Replace.getTypeId()).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(AttributeId.fromUUID(uuid), AttributeUpdateType.Replace, 1, 2));
    MergeSegmentOperation mergeSegmentOperation = new MergeSegmentOperation(1, 2, attributeUpdates);
    mergeSegmentOperation.setStreamSegmentOffset(123);
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, mergeSegmentOperation));
    Map<AttributeId, Long> attributes = new HashMap<>();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 2, 2, new StreamSegmentMapOperation(segmentProperties)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 3, 3, new StreamSegmentTruncateOperation(3, 3)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 4, 4, new UpdateAttributesOperation(4, attributeUpdates)));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 4: Add wrong inputs.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doThrow(NumberFormatException.class).doThrow(NullPointerException.class).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("wrong").doReturn("replace").doReturn("replace").when(command).getStringUserInput(Mockito.any());
    command.getDurableLogEditsFromUser();
}

Example 3 with DeleteSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation in project pravega by pravega.

the class DataRecoveryTest method testDurableLogRepairCommandExpectedLogOutput.

@Test
public void testDurableLogRepairCommandExpectedLogOutput() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId);
    val bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort()).with(BookKeeperConfig.BK_LEDGER_PATH, pravegaRunner.getBookKeeperRunner().getLedgerPath()).with(BookKeeperConfig.ZK_METADATA_PATH, pravegaRunner.getBookKeeperRunner().getLogMetaNamespace()).with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).build();
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down services, we assume that the cluster is in very bad shape in this test.
    pravegaRunner.shutDownControllerRunner();
    pravegaRunner.shutDownSegmentStoreRunner();
    // set Pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Execute basic command workflow for repairing DurableLog.
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    this.factory.initialize();
    // First, keep all the Operations of Container 0 in this list, so we can compare with the modified one.
    List<Operation> originalOperations = new ArrayList<>();
    @Cleanup DebugDurableDataLogWrapper wrapper = this.factory.createDebugLogWrapper(0);
    command.readDurableDataLogWithCustomCallback((op, entry) -> originalOperations.add(op), 0, wrapper.asReadOnly());
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // Second, add 2 operations, delete 1 operation, replace 1 operation.
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).when(command).confirmContinue();
    Mockito.doReturn(900L).doReturn(901L).doReturn(901L).doReturn(1L).doReturn(123L).doReturn(2L).doReturn(123L).doReturn(903L).doReturn(3L).doReturn(123L).doReturn(905L).doReturn(4L).doReturn(123L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").doReturn("add").doReturn("DeleteSegmentOperation").doReturn("DeleteSegmentOperation").doReturn("replace").doReturn("DeleteSegmentOperation").doReturn("add").doReturn("StreamSegmentSealOperation").when(command).getStringUserInput(Mockito.any());
    command.execute();
    List<Operation> originalOperationsEdited = new ArrayList<>();
    @Cleanup DebugDurableDataLogWrapper wrapperEdited = this.factory.createDebugLogWrapper(0);
    command.readDurableDataLogWithCustomCallback((op, entry) -> originalOperationsEdited.add(op), 0, wrapperEdited.asReadOnly());
    // OP-905 (now 907)
    for (int i = 899; i < 910; i++) {
        // Sequence numbers will defer between the original and edited logs. To do equality comparisons between
        // Operations in both logs, reset the sequence numbers (other fields should be the same).
        originalOperations.get(i).resetSequenceNumber(0);
        originalOperationsEdited.get(i).resetSequenceNumber(0);
    }
    Assert.assertNotEquals(originalOperations.get(899), originalOperationsEdited.get(899));
    Assert.assertTrue(originalOperationsEdited.get(899) instanceof DeleteSegmentOperation);
    Assert.assertTrue(originalOperationsEdited.get(900) instanceof DeleteSegmentOperation);
    Assert.assertEquals(originalOperations.get(900).toString(), originalOperationsEdited.get(901).toString());
    Assert.assertEquals(originalOperations.get(901).toString(), originalOperationsEdited.get(902).toString());
    Assert.assertTrue(originalOperationsEdited.get(903) instanceof DeleteSegmentOperation);
    Assert.assertEquals(originalOperations.get(903).toString(), originalOperationsEdited.get(904).toString());
    Assert.assertTrue(originalOperationsEdited.get(905) instanceof StreamSegmentSealOperation);
    Assert.assertEquals(originalOperations.get(904).toString(), originalOperationsEdited.get(906).toString());
    this.factory.close();
}

Example 4 with DeleteSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method preProcessOperation.

/**
 * Pre-processes the given Operation. See OperationMetadataUpdater.preProcessOperation for more details on behavior.
 *
 * @param operation The operation to pre-process.
 * @throws ContainerException     If the given operation was rejected given the current state of the container metadata.
 * @throws StreamSegmentException If the given operation was incompatible with the current state of the Segment.
 *                                For example: StreamSegmentNotExistsException, StreamSegmentSealedException or
 *                                StreamSegmentMergedException.
 */
void preProcessOperation(Operation operation) throws ContainerException, StreamSegmentException {
    checkNotSealed();
    if (operation instanceof SegmentOperation) {
        val segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        if (segmentMetadata.isDeleted()) {
            throw new StreamSegmentNotExistsException(segmentMetadata.getName());
        }
        if (operation instanceof StreamSegmentAppendOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentAppendOperation) operation);
        } else if (operation instanceof StreamSegmentSealOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentSealOperation) operation);
        } else if (operation instanceof MergeSegmentOperation) {
            MergeSegmentOperation mbe = (MergeSegmentOperation) operation;
            SegmentMetadataUpdateTransaction sourceMetadata = getSegmentUpdateTransaction(mbe.getSourceSegmentId());
            sourceMetadata.preProcessAsSourceSegment(mbe);
            segmentMetadata.preProcessAsTargetSegment(mbe, sourceMetadata);
        } else if (operation instanceof UpdateAttributesOperation) {
            segmentMetadata.preProcessOperation((UpdateAttributesOperation) operation);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentTruncateOperation) operation);
        } else if (operation instanceof DeleteSegmentOperation) {
            segmentMetadata.preProcessOperation((DeleteSegmentOperation) operation);
        }
    }
    if (operation instanceof MetadataCheckpointOperation) {
        // MetadataCheckpointOperations do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((MetadataCheckpointOperation) operation);
    } else if (operation instanceof StorageMetadataCheckpointOperation) {
        // StorageMetadataCheckpointOperation do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((StorageMetadataCheckpointOperation) operation);
    } else if (operation instanceof StreamSegmentMapOperation) {
        preProcessMetadataOperation((StreamSegmentMapOperation) operation);
    }
}

Example 5 with DeleteSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation in project pravega by pravega.

the class SegmentAggregatorTests method generateDeleteAndUpdateMetadata.

private StorageOperation generateDeleteAndUpdateMetadata(long segmentId, TestContext context) {
    UpdateableSegmentMetadata metadata = context.containerMetadata.getStreamSegmentMetadata(segmentId);
    metadata.markDeleted();
    DeleteSegmentOperation op = new DeleteSegmentOperation(segmentId);
    op.setStreamSegmentOffset(metadata.getLength());
    op.setSequenceNumber(context.containerMetadata.nextOperationSequenceNumber());
    return op;
}