Search in sources :

Example 1 with CommandArgs

use of io.pravega.cli.admin.CommandArgs in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationCorrectness.

@Test
public void testRepairLogEditOperationCorrectness() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = new DurableDataLogRepairCommand(args);
    // Check when no commands are available.
    command.checkDurableLogEdits(new ArrayList<>());
    // Check adding Edit Operations with sequence numbers lower than 0.
    AssertExtensions.assertThrows("Edit Operations should have initial sequence ids > 0.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, -1, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 0, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    // A Delete Edit Operation should have an initial sequence number lower than the final one.
    AssertExtensions.assertThrows("Edit Operations should have initial sequence ids > 0.", () -> command.checkDurableLogEdits(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 2, 2, null))), ex -> ex instanceof IllegalStateException);
    // Add one Add Edit and one Replace Edit on the same sequence number. This is expected to fail.
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 10, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    // We can have multiple Add Edit Operations on the same sequence number.
    command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0))));
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 10, 10, new DeleteSegmentOperation(0)), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.REPLACE_OPERATION, 10, 10, new DeleteSegmentOperation(0)))), ex -> ex instanceof IllegalStateException);
    AssertExtensions.assertThrows("Two non-Add Edit Operation on the same Sequence Number should not be accepted.", () -> command.checkDurableLogEdits(Arrays.asList(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 10, null), new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 5, 20, null))), ex -> ex instanceof IllegalStateException);
}
Also used : CommandArgs(io.pravega.cli.admin.CommandArgs) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) AdminCommandState(io.pravega.cli.admin.AdminCommandState) Test(org.junit.Test)

Example 2 with CommandArgs

use of io.pravega.cli.admin.CommandArgs in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationUserInput.

@Test
public void testRepairLogEditOperationUserInput() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Case 1: Input a Delete Edit Operation with wrong initial/final ids. Then retry with correct ids.
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").when(command).getStringUserInput(Mockito.any());
    Assert.assertEquals(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 2, null)), command.getDurableLogEditsFromUser());
    // Case 2: Input an Add Edit Operation with a wrong operation type. Then retry with correct operation type.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("wrong").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    DeleteSegmentOperation deleteOperationAdded = new DeleteSegmentOperation(1);
    List<DurableDataLogRepairCommand.LogEditOperation> editOps = new ArrayList<>();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 3: Create rest of operation types without payload (MergeSegmentOperation, StreamSegmentMapOperation, StreamSegmentTruncateOperation, UpdateAttributesOperation)
    long timestamp = System.currentTimeMillis();
    UUID uuid = UUID.randomUUID();
    editOps.clear();
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(2L).doReturn(1L).doReturn(2L).doReturn(123L).doReturn(2L).doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).doReturn(3L).doReturn(3L).doReturn(4L).doReturn(4L).doReturn(3L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("MergeSegmentOperation").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentMapOperation").doReturn("test").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentTruncateOperation").doReturn("add").doReturn("UpdateAttributesOperation").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn((int) AttributeUpdateType.Replace.getTypeId()).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(AttributeId.fromUUID(uuid), AttributeUpdateType.Replace, 1, 2));
    MergeSegmentOperation mergeSegmentOperation = new MergeSegmentOperation(1, 2, attributeUpdates);
    mergeSegmentOperation.setStreamSegmentOffset(123);
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, mergeSegmentOperation));
    Map<AttributeId, Long> attributes = new HashMap<>();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 2, 2, new StreamSegmentMapOperation(segmentProperties)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 3, 3, new StreamSegmentTruncateOperation(3, 3)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 4, 4, new UpdateAttributesOperation(4, attributeUpdates)));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 4: Add wrong inputs.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doThrow(NumberFormatException.class).doThrow(NullPointerException.class).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("wrong").doReturn("replace").doReturn("replace").when(command).getStringUserInput(Mockito.any());
    command.getDurableLogEditsFromUser();
}
Also used : CommandArgs(io.pravega.cli.admin.CommandArgs) AttributeUpdateCollection(io.pravega.segmentstore.contracts.AttributeUpdateCollection) AttributeUpdate(io.pravega.segmentstore.contracts.AttributeUpdate) UpdateAttributesOperation(io.pravega.segmentstore.server.logs.operations.UpdateAttributesOperation) HashMap(java.util.HashMap) AttributeId(io.pravega.segmentstore.contracts.AttributeId) ArrayList(java.util.ArrayList) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) ImmutableDate(io.pravega.common.util.ImmutableDate) StreamSegmentTruncateOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) UUID(java.util.UUID) AdminCommandState(io.pravega.cli.admin.AdminCommandState) Test(org.junit.Test)

Example 3 with CommandArgs

use of io.pravega.cli.admin.CommandArgs in project pravega by pravega.

the class DataRecoveryTest method testBasicDurableLogRepairCommand.

@Test
public void testBasicDurableLogRepairCommand() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId);
    val bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort()).with(BookKeeperConfig.BK_LEDGER_PATH, pravegaRunner.getBookKeeperRunner().getLedgerPath()).with(BookKeeperConfig.ZK_METADATA_PATH, pravegaRunner.getBookKeeperRunner().getLogMetaNamespace()).with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).build();
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down services, we assume that the cluster is in very bad shape in this test.
    pravegaRunner.shutDownControllerRunner();
    pravegaRunner.shutDownSegmentStoreRunner();
    // set Pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Execute basic command workflow for repairing DurableLog.
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // First execution, just exit when asking to disable the original log.
    command.execute();
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // The test will exercise editing the Container 0 log with an operation of each type.
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).when(command).confirmContinue();
    Mockito.doReturn(900L).doReturn(901L).doReturn(902L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    command.execute();
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // Now, re-execute the command to exercise the case in which there is an existing backup log.
    Mockito.doReturn(1).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).when(command).confirmContinue();
    Mockito.doReturn(900L).doReturn(901L).doReturn(902L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    command.execute();
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // Re-execute, now adding a replace operation and not destroying previous backup log.
    Mockito.doReturn(2).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(false).doReturn(true).when(command).confirmContinue();
    Mockito.doReturn(900L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("replace").doReturn("StreamSegmentSealOperation").doReturn("replace").doReturn("StreamSegmentSealOperation").when(command).getStringUserInput(Mockito.any());
    command.execute();
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // Do nothing if we find an existing backup log.
    Mockito.doReturn(3).when(command).getIntUserInput(Mockito.any());
    command.execute();
    DurableDataLogRepairCommand.descriptor();
}
Also used : lombok.val(lombok.val) CommandArgs(io.pravega.cli.admin.CommandArgs) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) Cleanup(lombok.Cleanup) TestUtils(io.pravega.cli.admin.utils.TestUtils) ByteArrayInputStream(java.io.ByteArrayInputStream) AdminCommandState(io.pravega.cli.admin.AdminCommandState) BookKeeperLogFactory(io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory) Test(org.junit.Test)

Example 4 with CommandArgs

use of io.pravega.cli.admin.CommandArgs in project pravega by pravega.

the class DataRecoveryTest method testForceMetadataOverWrite.

@Test
public void testForceMetadataOverWrite() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId);
    val bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort()).with(BookKeeperConfig.BK_LEDGER_PATH, pravegaRunner.getBookKeeperRunner().getLedgerPath()).with(BookKeeperConfig.ZK_METADATA_PATH, pravegaRunner.getBookKeeperRunner().getLogMetaNamespace()).with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).build();
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down services, we assume that the cluster is in very bad shape in this test.
    pravegaRunner.shutDownControllerRunner();
    pravegaRunner.shutDownSegmentStoreRunner();
    // set Pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Execute basic command workflow for repairing DurableLog.
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Test the DurableLogWrapper options to get, overwrite and destroy logs.
    @Cleanup val newFactory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    newFactory.initialize();
    @Cleanup DebugBookKeeperLogWrapper debugLogWrapper0 = newFactory.createDebugLogWrapper(0);
    int container0LogEntries = command.readDurableDataLogWithCustomCallback((a, b) -> {
    }, 0, debugLogWrapper0.asReadOnly());
    Assert.assertTrue(container0LogEntries > 0);
    ReadOnlyBookkeeperLogMetadata metadata0 = debugLogWrapper0.fetchMetadata();
    Assert.assertNotNull(metadata0);
    // Create a Repair log with some random content.
    @Cleanup DurableDataLog repairLog = newFactory.createDurableDataLog(this.factory.getRepairLogId());
    repairLog.initialize(TIMEOUT);
    repairLog.append(new CompositeByteArraySegment(new byte[0]), TIMEOUT).join();
    @Cleanup DebugBookKeeperLogWrapper debugLogWrapperRepair = newFactory.createDebugLogWrapper(0);
    // Overwrite metadata of repair container with metadata of container 0.
    debugLogWrapperRepair.forceMetadataOverWrite(metadata0);
    // Now the amount of log entries read should be equal to the ones of container 0.
    int newContainerRepairLogEntries = command.readDurableDataLogWithCustomCallback((a, b) -> {
    }, this.factory.getRepairLogId(), debugLogWrapperRepair.asReadOnly());
    ReadOnlyBookkeeperLogMetadata newMetadata1 = debugLogWrapperRepair.fetchMetadata();
    Assert.assertEquals(container0LogEntries, newContainerRepairLogEntries);
    Assert.assertEquals(metadata0.getLedgers(), newMetadata1.getLedgers());
    // Destroy contents of Container 0.
    debugLogWrapper0.deleteDurableLogMetadata();
    Assert.assertNull(debugLogWrapper0.fetchMetadata());
}
Also used : lombok.val(lombok.val) CommandArgs(io.pravega.cli.admin.CommandArgs) DurableDataLog(io.pravega.segmentstore.storage.DurableDataLog) ReadOnlyBookkeeperLogMetadata(io.pravega.segmentstore.storage.impl.bookkeeper.ReadOnlyBookkeeperLogMetadata) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) Cleanup(lombok.Cleanup) CompositeByteArraySegment(io.pravega.common.util.CompositeByteArraySegment) TestUtils(io.pravega.cli.admin.utils.TestUtils) DebugBookKeeperLogWrapper(io.pravega.segmentstore.storage.impl.bookkeeper.DebugBookKeeperLogWrapper) AdminCommandState(io.pravega.cli.admin.AdminCommandState) BookKeeperLogFactory(io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory) Test(org.junit.Test)

Example 5 with CommandArgs

use of io.pravega.cli.admin.CommandArgs in project pravega by pravega.

the class DataRecoveryTest method testDurableLogRepairCommandExpectedLogOutput.

@Test
public void testDurableLogRepairCommandExpectedLogOutput() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId);
    val bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort()).with(BookKeeperConfig.BK_LEDGER_PATH, pravegaRunner.getBookKeeperRunner().getLedgerPath()).with(BookKeeperConfig.ZK_METADATA_PATH, pravegaRunner.getBookKeeperRunner().getLogMetaNamespace()).with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).build();
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down services, we assume that the cluster is in very bad shape in this test.
    pravegaRunner.shutDownControllerRunner();
    pravegaRunner.shutDownSegmentStoreRunner();
    // set Pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Execute basic command workflow for repairing DurableLog.
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    this.factory.initialize();
    // First, keep all the Operations of Container 0 in this list, so we can compare with the modified one.
    List<Operation> originalOperations = new ArrayList<>();
    @Cleanup DebugDurableDataLogWrapper wrapper = this.factory.createDebugLogWrapper(0);
    command.readDurableDataLogWithCustomCallback((op, entry) -> originalOperations.add(op), 0, wrapper.asReadOnly());
    // Disable Original Log first.
    System.setIn(new ByteArrayInputStream("yes".getBytes()));
    TestUtils.executeCommand("bk disable 0", STATE.get());
    // Second, add 2 operations, delete 1 operation, replace 1 operation.
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).when(command).confirmContinue();
    Mockito.doReturn(900L).doReturn(901L).doReturn(901L).doReturn(1L).doReturn(123L).doReturn(2L).doReturn(123L).doReturn(903L).doReturn(3L).doReturn(123L).doReturn(905L).doReturn(4L).doReturn(123L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").doReturn("add").doReturn("DeleteSegmentOperation").doReturn("DeleteSegmentOperation").doReturn("replace").doReturn("DeleteSegmentOperation").doReturn("add").doReturn("StreamSegmentSealOperation").when(command).getStringUserInput(Mockito.any());
    command.execute();
    List<Operation> originalOperationsEdited = new ArrayList<>();
    @Cleanup DebugDurableDataLogWrapper wrapperEdited = this.factory.createDebugLogWrapper(0);
    command.readDurableDataLogWithCustomCallback((op, entry) -> originalOperationsEdited.add(op), 0, wrapperEdited.asReadOnly());
    // OP-905 (now 907)
    for (int i = 899; i < 910; i++) {
        // Sequence numbers will defer between the original and edited logs. To do equality comparisons between
        // Operations in both logs, reset the sequence numbers (other fields should be the same).
        originalOperations.get(i).resetSequenceNumber(0);
        originalOperationsEdited.get(i).resetSequenceNumber(0);
    }
    Assert.assertNotEquals(originalOperations.get(899), originalOperationsEdited.get(899));
    Assert.assertTrue(originalOperationsEdited.get(899) instanceof DeleteSegmentOperation);
    Assert.assertTrue(originalOperationsEdited.get(900) instanceof DeleteSegmentOperation);
    Assert.assertEquals(originalOperations.get(900).toString(), originalOperationsEdited.get(901).toString());
    Assert.assertEquals(originalOperations.get(901).toString(), originalOperationsEdited.get(902).toString());
    Assert.assertTrue(originalOperationsEdited.get(903) instanceof DeleteSegmentOperation);
    Assert.assertEquals(originalOperations.get(903).toString(), originalOperationsEdited.get(904).toString());
    Assert.assertTrue(originalOperationsEdited.get(905) instanceof StreamSegmentSealOperation);
    Assert.assertEquals(originalOperations.get(904).toString(), originalOperationsEdited.get(906).toString());
    this.factory.close();
}
Also used : lombok.val(lombok.val) CommandArgs(io.pravega.cli.admin.CommandArgs) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) ArrayList(java.util.ArrayList) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentTruncateOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) UpdateAttributesOperation(io.pravega.segmentstore.server.logs.operations.UpdateAttributesOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) Cleanup(lombok.Cleanup) TestUtils(io.pravega.cli.admin.utils.TestUtils) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) DebugDurableDataLogWrapper(io.pravega.segmentstore.storage.DebugDurableDataLogWrapper) ByteArrayInputStream(java.io.ByteArrayInputStream) AdminCommandState(io.pravega.cli.admin.AdminCommandState) BookKeeperLogFactory(io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory) Test(org.junit.Test)

Aggregations

CommandArgs (io.pravega.cli.admin.CommandArgs)18 Test (org.junit.Test)16 Properties (java.util.Properties)12 AdminCommandState (io.pravega.cli.admin.AdminCommandState)11 SegmentProperties (io.pravega.segmentstore.contracts.SegmentProperties)10 Cleanup (lombok.Cleanup)5 ByteArrayInputStream (java.io.ByteArrayInputStream)4 ArrayList (java.util.ArrayList)4 lombok.val (lombok.val)4 TestUtils (io.pravega.cli.admin.utils.TestUtils)3 AttributeUpdateCollection (io.pravega.segmentstore.contracts.AttributeUpdateCollection)3 DeleteSegmentOperation (io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation)3 BookKeeperLogFactory (io.pravega.segmentstore.storage.impl.bookkeeper.BookKeeperLogFactory)3 UUID (java.util.UUID)3 Parser (io.pravega.cli.admin.Parser)2 CompositeByteArraySegment (io.pravega.common.util.CompositeByteArraySegment)2 ImmutableDate (io.pravega.common.util.ImmutableDate)2 AttributeId (io.pravega.segmentstore.contracts.AttributeId)2 AttributeUpdate (io.pravega.segmentstore.contracts.AttributeUpdate)2 DataCorruptionException (io.pravega.segmentstore.server.DataCorruptionException)2