Examples with TestDurableDataLog io.pravega.segmentstore.server.TestDurableDataLog used on opensource projects

Example 6 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DurableLogTests method testRecoveryWithDisabledDataLog.

/**
 * Verifies the ability of hte DurableLog to recover (delayed start) using a disabled DurableDataLog. This verifies
 * the ability to shut down correctly while still waiting for the DataLog to become enabled as well as detecting that
 * it did become enabled and then resume normal operations.
 */
@Test
public void testRecoveryWithDisabledDataLog() throws Exception {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    // Write some data to the log. We'll read it later.
    HashSet<Long> streamSegmentIds;
    List<Operation> originalOperations;
    List<OperationWithCompletion> completionFutures;
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        completionFutures = processOperations(operations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
    }
    // Disable the DurableDataLog. This requires us to initialize the log, then disable it.
    metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertTrue("awaitOnline() returned an incomplete future.", Futures.isSuccessful(online));
        Assert.assertFalse("Not expecting an offline DurableLog.", durableLog.isOffline());
        dataLog.get().disable();
    }
    // Verify that the DurableLog starts properly and that all operations throw appropriate exceptions.
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertFalse("awaitOnline() returned a completed future.", online.isDone());
        Assert.assertTrue("Expecting an offline DurableLog.", durableLog.isOffline());
        // Verify all operations fail with the right exception.
        AssertExtensions.assertThrows("add() did not fail with the right exception when offline.", () -> durableLog.add(new ProbeOperation(), TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("read() did not fail with the right exception when offline.", () -> durableLog.read(0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("truncate() did not fail with the right exception when offline.", () -> durableLog.truncate(0, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("operationProcessingBarrier() did not fail with the right exception when offline.", () -> durableLog.operationProcessingBarrier(TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        // Verify we can also shut it down properly from this state.
        durableLog.stopAsync().awaitTerminated();
        Assert.assertTrue("awaitOnline() returned future did not fail when DurableLog shut down.", online.isCompletedExceptionally());
    }
    // Verify that, when the DurableDataLog becomes enabled, the DurableLog can pick up the change and resume normal operations.
    // Verify that the DurableLog starts properly and that all operations throw appropriate exceptions.
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertFalse("awaitOnline() returned a completed future.", online.isDone());
        // Enable the underlying data log and await for recovery to finish.
        dataLog.get().enable();
        online.get(START_RETRY_DELAY_MILLIS * 100, TimeUnit.MILLISECONDS);
        Assert.assertFalse("Not expecting an offline DurableLog after re-enabling.", durableLog.isOffline());
        // Verify we can still read the data that we wrote before the DataLog was disabled.
        List<Operation> recoveredOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        assertRecoveredOperationsMatch(originalOperations, recoveredOperations);
        performMetadataChecks(streamSegmentIds, new HashSet<>(), new HashMap<>(), completionFutures, metadata, false, false);
        performReadIndexChecks(completionFutures, readIndex);
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}

Example 7 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class OperationProcessorTests method testWithSingleProbeOperation.

/**
 * Tests the ability of the OperationProcessor to handle a single ProbeOperation (this is because it's a non-serializable
 * operation, so there is no commit to DurableDataLog - we need to verify the operation is properly completed in this
 * case).
 */
@Test
public void testWithSingleProbeOperation() throws Exception {
    @Cleanup TestContext context = new TestContext();
    // Generate some test data.
    ProbeOperation operation = new ProbeOperation();
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    OperationWithCompletion completionFuture = processOperations(Collections.singleton(operation), operationProcessor).get(0);
    // Wait for the ProbeOperation to complete (without exception). This is all we need to verify.
    completionFuture.completion.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Stop the processor.
    operationProcessor.stopAsync().awaitTerminated();
}

Example 8 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class OperationProcessorTests method testWithNoFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations in a failure-free environment.
 */
@Test
public void testWithNoFailures() throws Exception {
    int streamSegmentCount = 50;
    int transactionsPerStreamSegment = 2;
    int appendsPerStreamSegment = 20;
    boolean mergeTransactions = true;
    boolean sealStreamSegments = true;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data.
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    AbstractMap<Long, Long> transactions = createTransactionsInMetadata(streamSegmentIds, transactionsPerStreamSegment, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, transactions, appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, mergeTransactions, sealStreamSegments);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. If any of them failed, this will fail too and report the exception.
    OperationWithCompletion.allOf(completionFutures).join();
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, new HashSet<>(), transactions, completionFutures, context.metadata, mergeTransactions, sealStreamSegments);
    performReadIndexChecks(completionFutures, context.readIndex);
    operationProcessor.stopAsync().awaitTerminated();
}

Example 9 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class OperationProcessorTests method testWithDataLogFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations when there are DataLog write failures. The expected
 * outcome is that the OperationProcessor will auto-shutdown when such errors are encountered.
 */
@Test
public void testWithDataLogFailures() throws Exception {
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 80;
    // Fail (asynchronously) after X DataFrame commits (to DataLog).
    int failAfterCommits = 5;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    ErrorInjector<Exception> aSyncErrorInjector = new ErrorInjector<>(count -> count >= failAfterCommits, () -> new DurableDataLogException("intentional"));
    dataLog.setAppendErrorInjectors(null, aSyncErrorInjector);
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. We are expecting exceptions, so verify that we do.
    AssertExtensions.assertThrows("No operations failed.", OperationWithCompletion.allOf(completionFutures)::join, super::isExpectedExceptionForNonDataCorruption);
    // Wait for the OperationProcessor to shutdown with failure.
    ServiceListeners.awaitShutdown(operationProcessor, TIMEOUT, false);
    Assert.assertEquals("Expected the OperationProcessor to fail after DurableDataLogException encountered.", Service.State.FAILED, operationProcessor.state());
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, new HashSet<>(), new HashMap<>(), completionFutures, context.metadata, false, false);
    performReadIndexChecks(completionFutures, context.readIndex);
}

Example 10 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class OperationProcessorTests method testWithDataCorruptionFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations when a simulated DataCorruptionException
 * is generated.
 */
@Test
@SuppressWarnings("checkstyle:CyclomaticComplexity")
public void testWithDataCorruptionFailures() throws Exception {
    // If a DataCorruptionException is thrown for a particular Operation, the OperationQueueProcessor should
    // immediately shut down and stop accepting other ops.
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 80;
    // Fail Operation at index X.
    int failAtOperationIndex = 123;
    @Cleanup TestContext context = new TestContext();
    // Create a different state updater and Memory log - and use these throughout this test.
    CorruptedMemoryOperationLog corruptedMemoryLog = new CorruptedMemoryOperationLog(failAtOperationIndex);
    MemoryStateUpdater stateUpdater = new MemoryStateUpdater(corruptedMemoryLog, context.readIndex, Runnables.doNothing());
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for the store to fail (and make sure it failed).
    AssertExtensions.assertThrows("Operation Processor did not shut down with failure.", () -> ServiceListeners.awaitShutdown(operationProcessor, true), ex -> ex instanceof IllegalStateException);
    Assert.assertEquals("Unexpected service state after encountering DataCorruptionException.", Service.State.FAILED, operationProcessor.state());
    // Verify that the "right" operations failed, while the others succeeded.
    int successCount = 0;
    boolean encounteredFirstFailure = false;
    for (int i = 0; i < completionFutures.size(); i++) {
        OperationWithCompletion oc = completionFutures.get(i);
        if (!oc.operation.canSerialize()) {
            // Non-serializable operations (i.e., ProbeOperations always complete normally).
            continue;
        }
        // Once an operation failed (in our scenario), no other operation can succeed.
        if (encounteredFirstFailure) {
            Assert.assertTrue("Encountered successful operation after a failed operation: " + oc.operation, oc.completion.isCompletedExceptionally());
        }
        // with it, which is why it's hard to determine precisely what the first expected failed operation is.
        if (oc.completion.isCompletedExceptionally()) {
            // If we do find a failed one in this area, make sure it is failed with DataCorruptionException.
            AssertExtensions.assertThrows("Unexpected exception for failed Operation in the same DataFrame as intentionally failed operation.", oc.completion::join, super::isExpectedExceptionForDataCorruption);
            encounteredFirstFailure = true;
        } else {
            successCount++;
        }
    }
    AssertExtensions.assertGreaterThan("No operation succeeded.", 0, successCount);
    performLogOperationChecks(completionFutures, corruptedMemoryLog, dataLog, context.metadata, failAtOperationIndex - 1);
// There is no point in performing metadata checks. A DataCorruptionException means the Metadata (and the general
// state of the Container) is in an undefined state.
}