Search in sources :

Example 1 with ReadIndex

use of io.pravega.segmentstore.server.ReadIndex in project pravega by pravega.

the class StreamSegmentContainerTests method waitForOperationsInReadIndex.

/**
 * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog.
 * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to
 * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use
 * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it
 * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when
 * it is, it is guaranteed that everything prior to that has been committed.
 */
private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    String segmentName = "test" + System.nanoTime();
    container.createStreamSegment(segmentName, null, timer.getRemaining()).thenCompose(v -> container.append(segmentName, new byte[1], null, timer.getRemaining())).thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())).thenCompose(rr -> {
        ReadResultEntry rre = rr.next();
        rre.requestContent(TIMEOUT);
        return rre.getContent().thenRun(rr::close);
    }).thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}
Also used : Arrays(java.util.Arrays) Storage(io.pravega.segmentstore.storage.Storage) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) Cleanup(lombok.Cleanup) StorageWriterFactory(io.pravega.segmentstore.server.writer.StorageWriterFactory) Future(java.util.concurrent.Future) ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents) InMemoryStorageFactory(io.pravega.segmentstore.storage.mocks.InMemoryStorageFactory) Duration(java.time.Duration) Map(java.util.Map) AsyncReadResultProcessor(io.pravega.segmentstore.server.reading.AsyncReadResultProcessor) ContainerReadIndexFactory(io.pravega.segmentstore.server.reading.ContainerReadIndexFactory) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) DurableLogFactory(io.pravega.segmentstore.server.logs.DurableLogFactory) Attributes(io.pravega.segmentstore.contracts.Attributes) DurableLogConfig(io.pravega.segmentstore.server.logs.DurableLogConfig) Writer(io.pravega.segmentstore.server.Writer) SegmentContainerFactory(io.pravega.segmentstore.server.SegmentContainerFactory) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) Futures(io.pravega.common.concurrent.Futures) ByteArrayOutputStream(java.io.ByteArrayOutputStream) TooManyActiveSegmentsException(io.pravega.segmentstore.contracts.TooManyActiveSegmentsException) Exceptions(io.pravega.common.Exceptions) StorageFactory(io.pravega.segmentstore.storage.StorageFactory) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) Runnables(com.google.common.util.concurrent.Runnables) ReadIndexConfig(io.pravega.segmentstore.server.reading.ReadIndexConfig) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Timeout(org.junit.rules.Timeout) ConfigurationException(io.pravega.common.util.ConfigurationException) StreamHelpers(io.pravega.common.io.StreamHelpers) WriterFactory(io.pravega.segmentstore.server.WriterFactory) Properties(java.util.Properties) DurableDataLog(io.pravega.segmentstore.storage.DurableDataLog) Executor(java.util.concurrent.Executor) lombok.val(lombok.val) OperationLog(io.pravega.segmentstore.server.OperationLog) Test(org.junit.Test) Service(com.google.common.util.concurrent.Service) AtomicLong(java.util.concurrent.atomic.AtomicLong) OperationLogFactory(io.pravega.segmentstore.server.OperationLogFactory) SegmentContainer(io.pravega.segmentstore.server.SegmentContainer) Assert(org.junit.Assert) WriterConfig(io.pravega.segmentstore.server.writer.WriterConfig) SneakyThrows(lombok.SneakyThrows) AssertExtensions(io.pravega.test.common.AssertExtensions) RequiredArgsConstructor(lombok.RequiredArgsConstructor) TimeoutException(java.util.concurrent.TimeoutException) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) ReadIndexFactory(io.pravega.segmentstore.server.ReadIndexFactory) AttributeUpdate(io.pravega.segmentstore.contracts.AttributeUpdate) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) SegmentHandle(io.pravega.segmentstore.storage.SegmentHandle) AbstractService(com.google.common.util.concurrent.AbstractService) CacheFactory(io.pravega.segmentstore.storage.CacheFactory) ServiceListeners(io.pravega.segmentstore.server.ServiceListeners) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ReadResultEntryType(io.pravega.segmentstore.contracts.ReadResultEntryType) UUID(java.util.UUID) DataLogWriterNotPrimaryException(io.pravega.segmentstore.storage.DataLogWriterNotPrimaryException) Collectors(java.util.stream.Collectors) SegmentMetadataComparer(io.pravega.segmentstore.server.SegmentMetadataComparer) StreamSegmentNameUtils(io.pravega.shared.segment.StreamSegmentNameUtils) List(java.util.List) InMemoryCacheFactory(io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory) BadOffsetException(io.pravega.segmentstore.contracts.BadOffsetException) DurableDataLogFactory(io.pravega.segmentstore.storage.DurableDataLogFactory) ReadResult(io.pravega.segmentstore.contracts.ReadResult) Setter(lombok.Setter) Getter(lombok.Getter) ConfigHelpers(io.pravega.segmentstore.server.ConfigHelpers) AsyncStorageWrapper(io.pravega.segmentstore.storage.AsyncStorageWrapper) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) ExecutorService(java.util.concurrent.ExecutorService) ExecutorServiceHelpers.newScheduledThreadPool(io.pravega.common.concurrent.ExecutorServiceHelpers.newScheduledThreadPool) TimeoutTimer(io.pravega.common.TimeoutTimer) RollingStorage(io.pravega.segmentstore.storage.rolling.RollingStorage) IntentionalException(io.pravega.test.common.IntentionalException) StreamSegmentMergedException(io.pravega.segmentstore.contracts.StreamSegmentMergedException) TestReadResultHandler(io.pravega.segmentstore.server.reading.TestReadResultHandler) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Rule(org.junit.Rule) TypedProperties(io.pravega.common.util.TypedProperties) AttributeUpdateType(io.pravega.segmentstore.contracts.AttributeUpdateType) ReadIndex(io.pravega.segmentstore.server.ReadIndex) Collections(java.util.Collections) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) TimeoutTimer(io.pravega.common.TimeoutTimer)

Example 2 with ReadIndex

use of io.pravega.segmentstore.server.ReadIndex in project pravega by pravega.

the class DurableLogTests method testRecoveryFailures.

/**
 * Tests the DurableLog recovery process in a scenario when there are failures during the process
 * (these may or may not be DataCorruptionExceptions).
 */
@Test
public void testRecoveryFailures() throws Exception {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    // Fail DataLog reads after X reads.
    int failReadAfter = 2;
    // Setup a DurableLog and start it.
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    HashSet<Long> streamSegmentIds;
    List<OperationWithCompletion> completionFutures;
    // First DurableLog. We use this for generating data.
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Generate some test data (we need to do this after we started the DurableLog because in the process of
        // recovery, it wipes away all existing metadata).
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        // Process all generated operations and wait for them to complete
        completionFutures = processOperations(operations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery failure due to DataLog Failures.
    metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // Inject some artificial error into the DataLogRead after a few reads.
        ErrorInjector<Exception> readNextInjector = new ErrorInjector<>(count -> count > failReadAfter, () -> new DataLogNotAvailableException("intentional"));
        dataLog.get().setReadErrorInjectors(null, readNextInjector);
        // Verify the exception thrown from startAsync() is of the right kind. This exception will be wrapped in
        // multiple layers, so we need to dig deep into it.
        AssertExtensions.assertThrows("Recovery did not fail properly when expecting DurableDataLogException.", () -> durableLog.startAsync().awaitRunning(), ex -> {
            if (ex instanceof IllegalStateException) {
                ex = ex.getCause();
            }
            if (ex == null) {
                try {
                    // We need this to enter a FAILED state to get its failure cause.
                    durableLog.awaitTerminated();
                } catch (Exception ex2) {
                    ex = durableLog.failureCause();
                }
            }
            ex = Exceptions.unwrap(ex);
            return ex instanceof DataLogNotAvailableException && ex.getMessage().equals("intentional");
        });
    }
    // Recovery failure due to DataCorruptionException.
    metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // Reset error injectors to nothing.
        dataLog.get().setReadErrorInjectors(null, null);
        AtomicInteger readCounter = new AtomicInteger();
        dataLog.get().setReadInterceptor(readItem -> {
            if (readCounter.incrementAndGet() > failReadAfter && readItem.getLength() > DataFrame.MIN_ENTRY_LENGTH_NEEDED) {
                // Mangle with the payload and overwrite its contents with a DataFrame having a bogus
                // previous sequence number.
                DataFrame df = DataFrame.ofSize(readItem.getLength());
                df.seal();
                ArrayView serialization = df.getData();
                return new InjectedReadItem(serialization.getReader(), serialization.getLength(), readItem.getAddress());
            }
            return readItem;
        });
        // Verify the exception thrown from startAsync() is of the right kind. This exception will be wrapped in
        // multiple layers, so we need to dig deep into it.
        AssertExtensions.assertThrows("Recovery did not fail properly when expecting DataCorruptionException.", () -> durableLog.startAsync().awaitRunning(), ex -> {
            if (ex instanceof IllegalStateException) {
                ex = ex.getCause();
            }
            return Exceptions.unwrap(ex) instanceof DataCorruptionException;
        });
        // Verify that the underlying DurableDataLog has been disabled.
        val disabledDataLog = dataLogFactory.createDurableDataLog(CONTAINER_ID);
        AssertExtensions.assertThrows("DurableDataLog has not been disabled following a recovery failure with DataCorruptionException.", () -> disabledDataLog.initialize(TIMEOUT), ex -> ex instanceof DataLogDisabledException);
    }
}
Also used : TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) ProbeOperation(io.pravega.segmentstore.server.logs.operations.ProbeOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) Cleanup(lombok.Cleanup) DataLogDisabledException(io.pravega.segmentstore.storage.DataLogDisabledException) CacheManager(io.pravega.segmentstore.server.reading.CacheManager) lombok.val(lombok.val) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ErrorInjector(io.pravega.test.common.ErrorInjector) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) InMemoryCacheFactory(io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory) AtomicReference(java.util.concurrent.atomic.AtomicReference) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) TimeoutException(java.util.concurrent.TimeoutException) DataLogNotAvailableException(io.pravega.segmentstore.storage.DataLogNotAvailableException) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) CompletionException(java.util.concurrent.CompletionException) DataLogWriterNotPrimaryException(io.pravega.segmentstore.storage.DataLogWriterNotPrimaryException) StreamSegmentException(io.pravega.segmentstore.contracts.StreamSegmentException) DurableDataLogException(io.pravega.segmentstore.storage.DurableDataLogException) DataLogDisabledException(io.pravega.segmentstore.storage.DataLogDisabledException) IntentionalException(io.pravega.test.common.IntentionalException) IOException(java.io.IOException) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) DataLogNotAvailableException(io.pravega.segmentstore.storage.DataLogNotAvailableException) Storage(io.pravega.segmentstore.storage.Storage) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) ArrayView(io.pravega.common.util.ArrayView) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) Test(org.junit.Test)

Example 3 with ReadIndex

use of io.pravega.segmentstore.server.ReadIndex in project pravega by pravega.

the class DurableLogTests method testRecoveryWithNoFailures.

// endregion
// region Recovery
/**
 * Tests the DurableLog recovery process in a scenario when there are no failures during the process.
 */
@Test
public void testRecoveryWithNoFailures() throws Exception {
    int streamSegmentCount = 50;
    int transactionsPerStreamSegment = 2;
    int appendsPerStreamSegment = 20;
    boolean mergeTransactions = true;
    boolean sealStreamSegments = true;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    HashSet<Long> streamSegmentIds;
    AbstractMap<Long, Long> transactions;
    List<OperationWithCompletion> completionFutures;
    List<Operation> originalOperations;
    // First DurableLog. We use this for generating data.
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Generate some test data (we need to do this after we started the DurableLog because in the process of
        // recovery, it wipes away all existing metadata).
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        transactions = createTransactionsWithOperations(streamSegmentIds, transactionsPerStreamSegment, metadata, durableLog, storage);
        List<Operation> operations = generateOperations(streamSegmentIds, transactions, appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, mergeTransactions, sealStreamSegments);
        // Process all generated operations and wait for them to complete
        completionFutures = processOperations(operations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        // Get a list of all the operations, before recovery.
        originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Second DurableLog. We use this for recovery.
    metadata = new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        List<Operation> recoveredOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        assertRecoveredOperationsMatch(originalOperations, recoveredOperations);
        performMetadataChecks(streamSegmentIds, new HashSet<>(), transactions, completionFutures, metadata, mergeTransactions, sealStreamSegments);
        performReadIndexChecks(completionFutures, readIndex);
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}
Also used : MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) InMemoryCacheFactory(io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) ProbeOperation(io.pravega.segmentstore.server.logs.operations.ProbeOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) Cleanup(lombok.Cleanup) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) Storage(io.pravega.segmentstore.storage.Storage) CacheManager(io.pravega.segmentstore.server.reading.CacheManager) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) Test(org.junit.Test)

Example 4 with ReadIndex

use of io.pravega.segmentstore.server.ReadIndex in project pravega by pravega.

the class DurableLogTests method testTruncateWithoutRecovery.

// endregion
// region Truncation
/**
 * Tests the truncate() method without doing any recovery.
 */
@Test
public void testTruncateWithoutRecovery() {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    // Setup a DurableLog and start it.
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    AtomicReference<Boolean> truncationOccurred = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    @Cleanup ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
    // First DurableLog. We use this for generating data.
    try (DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Hook up a listener to figure out when truncation actually happens.
        dataLog.get().setTruncateCallback(seqNo -> truncationOccurred.set(true));
        // Generate some test data (we need to do this after we started the DurableLog because in the process of
        // recovery, it wipes away all existing metadata).
        HashSet<Long> streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        List<Operation> queuedOperations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        // Add one of these at the end to ensure we can truncate everything.
        queuedOperations.add(new MetadataCheckpointOperation());
        List<OperationWithCompletion> completionFutures = processOperations(queuedOperations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        // Get a list of all the operations, before truncation.
        List<Operation> originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        boolean fullTruncationPossible = false;
        // At the end, verify all operations and all entries in the DataLog were truncated.
        for (int i = 0; i < originalOperations.size(); i++) {
            Operation currentOperation = originalOperations.get(i);
            truncationOccurred.set(false);
            if (currentOperation instanceof MetadataCheckpointOperation) {
                // Need to figure out if the operation we're about to truncate to is actually the first in the log;
                // in that case, we should not be expecting any truncation.
                boolean isTruncationPointFirstOperation = durableLog.read(-1, 1, TIMEOUT).join().next() instanceof MetadataCheckpointOperation;
                // Perform the truncation.
                durableLog.truncate(currentOperation.getSequenceNumber(), TIMEOUT).join();
                if (!isTruncationPointFirstOperation) {
                    Assert.assertTrue("No truncation occurred even though a valid Truncation Point was passed: " + currentOperation.getSequenceNumber(), truncationOccurred.get());
                }
                // Verify all operations up to, and including this one have been removed.
                Iterator<Operation> reader = durableLog.read(-1, 2, TIMEOUT).join();
                Assert.assertTrue("Not expecting an empty log after truncating an operation (a MetadataCheckpoint must always exist).", reader.hasNext());
                verifyFirstItemIsMetadataCheckpoint(reader);
                if (i < originalOperations.size() - 1) {
                    Operation firstOp = reader.next();
                    OperationComparer.DEFAULT.assertEquals(String.format("Unexpected first operation after truncating SeqNo %d.", currentOperation.getSequenceNumber()), originalOperations.get(i + 1), firstOp);
                } else {
                    // Sometimes the Truncation Point is on the same DataFrame as other data, and it's the last DataFrame;
                    // In that case, it cannot be truncated, since truncating the frame would mean losing the Checkpoint as well.
                    fullTruncationPossible = !reader.hasNext();
                }
            } else {
                // Verify we are not allowed to truncate on non-valid Truncation Points.
                AssertExtensions.assertThrows("DurableLog allowed truncation on a non-MetadataCheckpointOperation.", () -> durableLog.truncate(currentOperation.getSequenceNumber(), TIMEOUT), ex -> ex instanceof IllegalArgumentException);
                // Verify the Operation Log is still intact.
                Iterator<Operation> reader = durableLog.read(-1, 1, TIMEOUT).join();
                Assert.assertTrue("No elements left in the log even though no truncation occurred.", reader.hasNext());
                Operation firstOp = reader.next();
                AssertExtensions.assertLessThanOrEqual("It appears that Operations were removed from the Log even though no truncation happened.", currentOperation.getSequenceNumber(), firstOp.getSequenceNumber());
            }
        }
        // Verify that we can still queue operations to the DurableLog and they can be read.
        // In this case we'll just queue some StreamSegmentMapOperations.
        StreamSegmentMapOperation newOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name("foo").build());
        if (!fullTruncationPossible) {
            // We were not able to do a full truncation before. Do one now, since we are guaranteed to have a new DataFrame available.
            MetadataCheckpointOperation lastCheckpoint = new MetadataCheckpointOperation();
            durableLog.add(lastCheckpoint, TIMEOUT).join();
            durableLog.truncate(lastCheckpoint.getSequenceNumber(), TIMEOUT).join();
        }
        durableLog.add(newOp, TIMEOUT).join();
        // Full Checkpoint + Storage Checkpoint (auto-added)+ new op
        final int expectedOperationCount = 3;
        List<Operation> newOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        Assert.assertEquals("Unexpected number of operations added after full truncation.", expectedOperationCount, newOperations.size());
        Assert.assertTrue("Expecting the first operation after full truncation to be a MetadataCheckpointOperation.", newOperations.get(0) instanceof MetadataCheckpointOperation);
        Assert.assertTrue("Expecting a StorageMetadataCheckpointOperation to be auto-added after full truncation.", newOperations.get(1) instanceof StorageMetadataCheckpointOperation);
        Assert.assertEquals("Unexpected Operation encountered after full truncation.", newOp, newOperations.get(2));
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}
Also used : TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) ProbeOperation(io.pravega.segmentstore.server.logs.operations.ProbeOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) Cleanup(lombok.Cleanup) CacheManager(io.pravega.segmentstore.server.reading.CacheManager) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) InMemoryCacheFactory(io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory) AtomicReference(java.util.concurrent.atomic.AtomicReference) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) Storage(io.pravega.segmentstore.storage.Storage) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) Test(org.junit.Test)

Example 5 with ReadIndex

use of io.pravega.segmentstore.server.ReadIndex in project pravega by pravega.

the class DurableLogTests method testRecoveryWithDisabledDataLog.

/**
 * Verifies the ability of hte DurableLog to recover (delayed start) using a disabled DurableDataLog. This verifies
 * the ability to shut down correctly while still waiting for the DataLog to become enabled as well as detecting that
 * it did become enabled and then resume normal operations.
 */
@Test
public void testRecoveryWithDisabledDataLog() throws Exception {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    // Write some data to the log. We'll read it later.
    HashSet<Long> streamSegmentIds;
    List<Operation> originalOperations;
    List<OperationWithCompletion> completionFutures;
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        completionFutures = processOperations(operations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
    }
    // Disable the DurableDataLog. This requires us to initialize the log, then disable it.
    metadata = new MetadataBuilder(CONTAINER_ID).build();
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertTrue("awaitOnline() returned an incomplete future.", Futures.isSuccessful(online));
        Assert.assertFalse("Not expecting an offline DurableLog.", durableLog.isOffline());
        dataLog.get().disable();
    }
    // Verify that the DurableLog starts properly and that all operations throw appropriate exceptions.
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertFalse("awaitOnline() returned a completed future.", online.isDone());
        Assert.assertTrue("Expecting an offline DurableLog.", durableLog.isOffline());
        // Verify all operations fail with the right exception.
        AssertExtensions.assertThrows("add() did not fail with the right exception when offline.", () -> durableLog.add(new ProbeOperation(), TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("read() did not fail with the right exception when offline.", () -> durableLog.read(0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("truncate() did not fail with the right exception when offline.", () -> durableLog.truncate(0, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("operationProcessingBarrier() did not fail with the right exception when offline.", () -> durableLog.operationProcessingBarrier(TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        // Verify we can also shut it down properly from this state.
        durableLog.stopAsync().awaitTerminated();
        Assert.assertTrue("awaitOnline() returned future did not fail when DurableLog shut down.", online.isCompletedExceptionally());
    }
    // Verify that, when the DurableDataLog becomes enabled, the DurableLog can pick up the change and resume normal operations.
    // Verify that the DurableLog starts properly and that all operations throw appropriate exceptions.
    dataLog.set(null);
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        // DurableLog should start properly.
        durableLog.startAsync().awaitRunning();
        CompletableFuture<Void> online = durableLog.awaitOnline();
        Assert.assertFalse("awaitOnline() returned a completed future.", online.isDone());
        // Enable the underlying data log and await for recovery to finish.
        dataLog.get().enable();
        online.get(START_RETRY_DELAY_MILLIS * 100, TimeUnit.MILLISECONDS);
        Assert.assertFalse("Not expecting an offline DurableLog after re-enabling.", durableLog.isOffline());
        // Verify we can still read the data that we wrote before the DataLog was disabled.
        List<Operation> recoveredOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        assertRecoveredOperationsMatch(originalOperations, recoveredOperations);
        performMetadataChecks(streamSegmentIds, new HashSet<>(), new HashMap<>(), completionFutures, metadata, false, false);
        performReadIndexChecks(completionFutures, readIndex);
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}
Also used : TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) ProbeOperation(io.pravega.segmentstore.server.logs.operations.ProbeOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) Cleanup(lombok.Cleanup) CacheManager(io.pravega.segmentstore.server.reading.CacheManager) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) InMemoryCacheFactory(io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory) AtomicReference(java.util.concurrent.atomic.AtomicReference) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) Storage(io.pravega.segmentstore.storage.Storage) ProbeOperation(io.pravega.segmentstore.server.logs.operations.ProbeOperation) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) Test(org.junit.Test)

Aggregations

ReadIndex (io.pravega.segmentstore.server.ReadIndex)8 Test (org.junit.Test)8 MetadataBuilder (io.pravega.segmentstore.server.MetadataBuilder)7 TestDurableDataLogFactory (io.pravega.segmentstore.server.TestDurableDataLogFactory)7 UpdateableContainerMetadata (io.pravega.segmentstore.server.UpdateableContainerMetadata)7 StreamSegmentMapOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation)7 Storage (io.pravega.segmentstore.storage.Storage)7 InMemoryCacheFactory (io.pravega.segmentstore.storage.mocks.InMemoryCacheFactory)7 InMemoryDurableDataLogFactory (io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory)7 Cleanup (lombok.Cleanup)7 Operation (io.pravega.segmentstore.server.logs.operations.Operation)6 StorageOperation (io.pravega.segmentstore.server.logs.operations.StorageOperation)6 StreamSegmentAppendOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation)6 CacheManager (io.pravega.segmentstore.server.reading.CacheManager)6 ContainerReadIndex (io.pravega.segmentstore.server.reading.ContainerReadIndex)6 MetadataCheckpointOperation (io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation)5 ProbeOperation (io.pravega.segmentstore.server.logs.operations.ProbeOperation)5 StorageMetadataCheckpointOperation (io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation)5 AtomicReference (java.util.concurrent.atomic.AtomicReference)5 TestDurableDataLog (io.pravega.segmentstore.server.TestDurableDataLog)4