Examples with DurableDataLog io.pravega.segmentstore.storage.DurableDataLog used on opensource projects

Example 1 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, SequencedItemList<Operation> memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).filter(Operation::canSerialize).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    if (successfulOps.size() > 0) {
        // Writing to the memory log is asynchronous and we don't have any callbacks to know when it was written to.
        // We check periodically until the last item has been written.
        await(() -> memoryLog.read(successfulOps.get(successfulOps.size() - 1).getSequenceNumber() - 1, 1).hasNext(), 10);
    }
    Iterator<Operation> memoryLogIterator = memoryLog.read(-1, operations.size() + 1);
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        // Check truncation markers if this is the last Operation to be written.
        LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), dataFrameAddress);
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), dataFrameAddress);
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}

Example 2 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class DurableDataLogRepairCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(1);
    int containerId = getIntArg(0);
    val bkConfig = getCommandArgs().getState().getConfigBuilder().include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, getServiceConfig().getZkURL())).build().getConfig(BookKeeperConfig::builder);
    @Cleanup val zkClient = createZKClient();
    @Cleanup DurableDataLogFactory dataLogFactory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor());
    dataLogFactory.initialize();
    // Open the Original Log in read-only mode.
    @Cleanup val originalDataLog = dataLogFactory.createDebugLogWrapper(containerId);
    // Check if the Original Log is disabled.
    if (originalDataLog.fetchMetadata().isEnabled()) {
        output("Original DurableLog is enabled. Repairs can only be done on disabled logs, exiting.");
        return;
    }
    // Make sure that the reserved id for Backup log is free before making any further progress.
    boolean createNewBackupLog = true;
    if (existsBackupLog(dataLogFactory)) {
        output("We found data in the Backup log, probably from a previous repair operation (or someone else running the same command at the same time). " + "You have three options: 1) Delete existing Backup Log and start a new repair process, " + "2) Keep existing Backup Log and re-use it for the current repair (i.e., skip creating a new Backup Log), " + "3) Quit.");
        switch(getIntUserInput("Select an option: [1|2|3]")) {
            case 1:
                // Delete everything related to the old Backup Log.
                try (DebugDurableDataLogWrapper backupDataLogDebugLogWrapper = dataLogFactory.createDebugLogWrapper(dataLogFactory.getBackupLogId())) {
                    backupDataLogDebugLogWrapper.deleteDurableLogMetadata();
                }
                break;
            case 2:
                // Keeping existing Backup Log, so not creating a new one.
                createNewBackupLog = false;
                break;
            default:
                output("Not doing anything with existing Backup Log this time.");
                return;
        }
    }
    // Create a new Backup Log if there wasn't any or if we removed the existing one.
    if (createNewBackupLog) {
        createBackupLog(dataLogFactory, containerId, originalDataLog);
    }
    int backupLogReadOperations = validateBackupLog(dataLogFactory, containerId, originalDataLog, createNewBackupLog);
    // Get user input of operations to skip, replace, or delete.
    List<LogEditOperation> durableLogEdits = getDurableLogEditsFromUser();
    // Show the edits to be committed to the original durable log so the user can confirm.
    output("The following edits will be used to edit the Original Log:");
    durableLogEdits.forEach(e -> output(e.toString()));
    output("Original DurableLog has been backed up correctly. Ready to apply admin-provided changes to the Original Log.");
    if (!confirmContinue()) {
        output("Not editing Original DurableLog this time. A Backup Log has been left during the process and you " + "will find it the next time this command gets executed.");
        return;
    }
    // Ensure that the Repair Log is going to start from a clean state.
    output("Deleting existing medatadata from Repair Log (if any)");
    try (val editedLogWrapper = dataLogFactory.createDebugLogWrapper(dataLogFactory.getRepairLogId())) {
        editedLogWrapper.deleteDurableLogMetadata();
    } catch (DurableDataLogException e) {
        if (e.getCause() instanceof KeeperException.NoNodeException) {
            output("Repair Log does not exist, so nothing to delete.");
        } else {
            outputError("Error happened while attempting to cleanup Repair Log metadata.");
            outputException(e);
        }
    }
    // that will write the edited contents into the Repair Log.
    try (DurableDataLog editedDataLog = dataLogFactory.createDurableDataLog(dataLogFactory.getRepairLogId());
        EditingLogProcessor logEditState = new EditingLogProcessor(editedDataLog, durableLogEdits, getCommandArgs().getState().getExecutor());
        DurableDataLog backupDataLog = dataLogFactory.createDebugLogWrapper(dataLogFactory.getBackupLogId()).asReadOnly()) {
        editedDataLog.initialize(TIMEOUT);
        readDurableDataLogWithCustomCallback(logEditState, dataLogFactory.getBackupLogId(), backupDataLog);
        Preconditions.checkState(!logEditState.isFailed);
        // After the edition has completed, we need to disable it before the metadata overwrite.
        editedDataLog.disable();
    } catch (Exception ex) {
        outputError("There have been errors while creating the edited version of the DurableLog.");
        outputException(ex);
        throw ex;
    }
    // Validate the contents of the newly created Repair Log.
    int editedDurableLogOperations = validateRepairLog(dataLogFactory, backupLogReadOperations, durableLogEdits);
    // Overwrite the original DurableLog metadata with the edited DurableLog metadata.
    try (val editedLogWrapper = dataLogFactory.createDebugLogWrapper(dataLogFactory.getRepairLogId())) {
        output("Original DurableLog Metadata: " + originalDataLog.fetchMetadata());
        output("Edited DurableLog Metadata: " + editedLogWrapper.fetchMetadata());
        originalDataLog.forceMetadataOverWrite(editedLogWrapper.fetchMetadata());
        output("New Original DurableLog Metadata (after replacement): " + originalDataLog.fetchMetadata());
    }
    // Read the edited contents that are now reachable from the original log id.
    try (val editedLogWrapper = dataLogFactory.createDebugLogWrapper(dataLogFactory.getRepairLogId())) {
        int finalEditedLogReadOps = readDurableDataLogWithCustomCallback((op, list) -> output("Original Log Operations after repair: " + op), containerId, editedLogWrapper.asReadOnly());
        output("Original DurableLog operations read (after editing): " + finalEditedLogReadOps);
        Preconditions.checkState(editedDurableLogOperations == finalEditedLogReadOps, "Repair Log operations not matching before (" + editedDurableLogOperations + ") and after the metadata overwrite (" + finalEditedLogReadOps + ")");
    } catch (Exception ex) {
        outputError("Problem reading Original DurableLog after editing.");
        outputException(ex);
    }
    output("Process completed successfully! (You still need to enable the Durable Log so Pravega can use it)");
}

Example 3 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class DurableDataLogRepairCommand method validateRepairLog.

private int validateRepairLog(DurableDataLogFactory dataLogFactory, int backupLogReadOperations, List<LogEditOperation> durableLogEdits) throws Exception {
    @Cleanup DurableDataLog editedDebugDataLogReadOnly = dataLogFactory.createDebugLogWrapper(dataLogFactory.getRepairLogId()).asReadOnly();
    int editedDurableLogOperations = readDurableDataLogWithCustomCallback((op, list) -> output("Repair Log Operations: " + op), dataLogFactory.getRepairLogId(), editedDebugDataLogReadOnly);
    output("Edited DurableLog Operations read: " + editedDurableLogOperations);
    long expectedEditedLogOperations = backupLogReadOperations + durableLogEdits.stream().filter(edit -> edit.type.equals(LogEditType.ADD_OPERATION)).count() - durableLogEdits.stream().filter(edit -> edit.type.equals(LogEditType.DELETE_OPERATION)).map(edit -> edit.finalOperationId - edit.initialOperationId).reduce(Long::sum).orElse(0L);
    Preconditions.checkState(expectedEditedLogOperations == editedDurableLogOperations, "Expected (" + expectedEditedLogOperations + ") and actual (" + editedDurableLogOperations + ") operations in Edited Log do not match");
    return editedDurableLogOperations;
}

Example 4 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class DataRecoveryTest method testForceMetadataOverWrite.

@Test
public void testForceMetadataOverWrite() throws Exception {
    int instanceId = 0;
    int bookieCount = 3;
    int containerCount = 1;
    @Cleanup TestUtils.PravegaRunner pravegaRunner = new TestUtils.PravegaRunner(bookieCount, containerCount);
    pravegaRunner.startBookKeeperRunner(instanceId);
    val bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort()).with(BookKeeperConfig.BK_LEDGER_PATH, pravegaRunner.getBookKeeperRunner().getLedgerPath()).with(BookKeeperConfig.ZK_METADATA_PATH, pravegaRunner.getBookKeeperRunner().getLogMetaNamespace()).with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).build();
    this.factory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    pravegaRunner.startControllerAndSegmentStore(this.storageFactory, this.factory);
    String streamName = "testDataRecoveryCommand";
    TestUtils.createScopeStream(pravegaRunner.getControllerRunner().getController(), SCOPE, streamName, config);
    try (val clientRunner = new TestUtils.ClientRunner(pravegaRunner.getControllerRunner(), SCOPE)) {
        // Write events to the streams.
        TestUtils.writeEvents(streamName, clientRunner.getClientFactory());
    }
    // Shut down services, we assume that the cluster is in very bad shape in this test.
    pravegaRunner.shutDownControllerRunner();
    pravegaRunner.shutDownSegmentStoreRunner();
    // set Pravega properties for the test
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.storage.impl.name", "FILESYSTEM");
    pravegaProperties.setProperty("pravegaservice.storage.layout", "ROLLING_STORAGE");
    pravegaProperties.setProperty("pravegaservice.zk.connect.uri", "localhost:" + pravegaRunner.getBookKeeperRunner().getBkPort());
    pravegaProperties.setProperty("bookkeeper.ledger.path", pravegaRunner.getBookKeeperRunner().getLedgerPath());
    pravegaProperties.setProperty("bookkeeper.zk.metadata.path", pravegaRunner.getBookKeeperRunner().getLogMetaNamespace());
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    pravegaProperties.setProperty("filesystem.root", this.baseDir.getAbsolutePath());
    STATE.get().getConfigBuilder().include(pravegaProperties);
    // Execute basic command workflow for repairing DurableLog.
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Test the DurableLogWrapper options to get, overwrite and destroy logs.
    @Cleanup val newFactory = new BookKeeperLogFactory(bkConfig, pravegaRunner.getBookKeeperRunner().getZkClient().get(), this.executorService());
    newFactory.initialize();
    @Cleanup DebugBookKeeperLogWrapper debugLogWrapper0 = newFactory.createDebugLogWrapper(0);
    int container0LogEntries = command.readDurableDataLogWithCustomCallback((a, b) -> {
    }, 0, debugLogWrapper0.asReadOnly());
    Assert.assertTrue(container0LogEntries > 0);
    ReadOnlyBookkeeperLogMetadata metadata0 = debugLogWrapper0.fetchMetadata();
    Assert.assertNotNull(metadata0);
    // Create a Repair log with some random content.
    @Cleanup DurableDataLog repairLog = newFactory.createDurableDataLog(this.factory.getRepairLogId());
    repairLog.initialize(TIMEOUT);
    repairLog.append(new CompositeByteArraySegment(new byte[0]), TIMEOUT).join();
    @Cleanup DebugBookKeeperLogWrapper debugLogWrapperRepair = newFactory.createDebugLogWrapper(0);
    // Overwrite metadata of repair container with metadata of container 0.
    debugLogWrapperRepair.forceMetadataOverWrite(metadata0);
    // Now the amount of log entries read should be equal to the ones of container 0.
    int newContainerRepairLogEntries = command.readDurableDataLogWithCustomCallback((a, b) -> {
    }, this.factory.getRepairLogId(), debugLogWrapperRepair.asReadOnly());
    ReadOnlyBookkeeperLogMetadata newMetadata1 = debugLogWrapperRepair.fetchMetadata();
    Assert.assertEquals(container0LogEntries, newContainerRepairLogEntries);
    Assert.assertEquals(metadata0.getLedgers(), newMetadata1.getLedgers());
    // Destroy contents of Container 0.
    debugLogWrapper0.deleteDurableLogMetadata();
    Assert.assertNull(debugLogWrapper0.fetchMetadata());
}

Example 5 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class BookkeeperCommandsTest method createLedgerInBookkeeperTestCluster.

private void createLedgerInBookkeeperTestCluster(int logId) throws Exception {
    BookKeeperConfig bookKeeperConfig = BookKeeperConfig.builder().with(BookKeeperConfig.BK_ENSEMBLE_SIZE, 1).with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 1).with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 1).with(BookKeeperConfig.ZK_METADATA_PATH, "ledgers").with(BookKeeperConfig.BK_LEDGER_PATH, "/ledgers").with(BookKeeperConfig.ZK_ADDRESS, zkUtil.getZooKeeperConnectString()).build();
    @Cleanup CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(zkUtil.getZooKeeperConnectString(), new RetryOneTime(5000));
    curatorFramework.start();
    @Cleanup("shutdownNow") ScheduledExecutorService executorService = ExecutorServiceHelpers.newScheduledThreadPool(1, "bk-test");
    @Cleanup BookKeeperLogFactory bookKeeperLogFactory = new BookKeeperLogFactory(bookKeeperConfig, curatorFramework, executorService);
    bookKeeperLogFactory.initialize();
    @Cleanup DurableDataLog log = bookKeeperLogFactory.createDurableDataLog(logId);
    log.initialize(Duration.ofSeconds(5));
}