Search in sources :

Example 6 with ContainerReadIndex

use of io.pravega.segmentstore.server.reading.ContainerReadIndex in project pravega by pravega.

the class DurableLogTests method testRecoveryWithIncrementalCheckpoints.

/**
 * Tests the DurableLog recovery process when there are multiple {@link MetadataCheckpointOperation}s added, with each
 * such checkpoint including information about evicted segments or segments which had their storage state modified.
 */
@Test
public void testRecoveryWithIncrementalCheckpoints() throws Exception {
    final int streamSegmentCount = 50;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    // First DurableLog. We use this for generating data.
    val metadata1 = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
    @Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
    List<Long> deletedIds;
    Set<Long> evictIds;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create some segments.
        val segmentIds = new ArrayList<>(createStreamSegmentsWithOperations(streamSegmentCount, durableLog));
        deletedIds = segmentIds.subList(0, 5);
        val mergedFromIds = segmentIds.subList(5, 10);
        // Must be same length as mergeFrom
        val mergedToIds = segmentIds.subList(10, 15);
        evictIds = new HashSet<>(segmentIds.subList(15, 20));
        val changeStorageStateIds = segmentIds.subList(20, segmentIds.size() - 5);
        // Append something to each segment.
        for (val segmentId : segmentIds) {
            if (!evictIds.contains(segmentId)) {
                durableLog.add(new StreamSegmentAppendOperation(segmentId, generateAppendData((int) (long) segmentId), null), OperationPriority.Normal, TIMEOUT).join();
            }
        }
        // Checkpoint 1.
        durableLog.checkpoint(TIMEOUT).join();
        // Delete some segments.
        for (val segmentId : deletedIds) {
            durableLog.add(new DeleteSegmentOperation(segmentId), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 2.
        durableLog.checkpoint(TIMEOUT).join();
        // Merge some segments.
        for (int i = 0; i < mergedFromIds.size(); i++) {
            durableLog.add(new StreamSegmentSealOperation(mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
            durableLog.add(new MergeSegmentOperation(mergedToIds.get(i), mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 3.
        durableLog.checkpoint(TIMEOUT).join();
        // Evict some segments.
        val evictableContainerMetadata = (EvictableMetadata) metadata1;
        metadata1.removeTruncationMarkers(metadata1.getOperationSequenceNumber());
        val toEvict = evictableContainerMetadata.getEvictionCandidates(Integer.MAX_VALUE, segmentIds.size()).stream().filter(m -> evictIds.contains(m.getId())).collect(Collectors.toList());
        val evicted = evictableContainerMetadata.cleanup(toEvict, Integer.MAX_VALUE);
        AssertExtensions.assertContainsSameElements("", evictIds, evicted.stream().map(SegmentMetadata::getId).collect(Collectors.toList()));
        // Checkpoint 4.
        durableLog.checkpoint(TIMEOUT).join();
        // Update storage state for some segments.
        for (val segmentId : changeStorageStateIds) {
            val sm = metadata1.getStreamSegmentMetadata(segmentId);
            if (segmentId % 3 == 0) {
                sm.setStorageLength(sm.getLength());
            }
            if (segmentId % 4 == 0) {
                sm.markSealed();
                sm.markSealedInStorage();
            }
            if (segmentId % 5 == 0) {
                sm.markDeleted();
                sm.markDeletedInStorage();
            }
        }
        // Checkpoint 5.
        durableLog.checkpoint(TIMEOUT).join();
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Second DurableLog. We use this for recovery.
    val metadata2 = new MetadataBuilder(CONTAINER_ID).build();
    try (ContainerReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Validate metadata matches.
        val expectedSegmentIds = metadata1.getAllStreamSegmentIds();
        val actualSegmentIds = metadata2.getAllStreamSegmentIds();
        AssertExtensions.assertContainsSameElements("Unexpected set of recovered segments. Only Active segments expected to have been recovered.", expectedSegmentIds, actualSegmentIds);
        val expectedSegments = expectedSegmentIds.stream().sorted().map(metadata1::getStreamSegmentMetadata).collect(Collectors.toList());
        val actualSegments = actualSegmentIds.stream().sorted().map(metadata2::getStreamSegmentMetadata).collect(Collectors.toList());
        for (int i = 0; i < expectedSegments.size(); i++) {
            val e = expectedSegments.get(i);
            val a = actualSegments.get(i);
            SegmentMetadataComparer.assertEquals("Recovered segment metadata mismatch", e, a);
        }
        // Validate read index is as it should. Here, we can only check if the read indices for evicted segments are
        // no longer loaded; we do more thorough checks in the ContainerReadIndexTests suite.
        Streams.concat(evictIds.stream(), deletedIds.stream()).forEach(segmentId -> Assert.assertNull("Not expecting a read index for an evicted or deleted segment.", readIndex.getIndex(segmentId)));
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}
Also used : Storage(io.pravega.segmentstore.storage.Storage) StreamSegmentInformation(io.pravega.segmentstore.contracts.StreamSegmentInformation) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) OperationPriority(io.pravega.segmentstore.server.logs.operations.OperationPriority) SneakyThrows(lombok.SneakyThrows) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) AssertExtensions(io.pravega.test.common.AssertExtensions) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) TimeoutException(java.util.concurrent.TimeoutException) Cleanup(lombok.Cleanup) LogAddress(io.pravega.segmentstore.storage.LogAddress) UpdateableSegmentMetadata(io.pravega.segmentstore.server.UpdateableSegmentMetadata) DataLogNotAvailableException(io.pravega.segmentstore.storage.DataLogNotAvailableException) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CheckpointOperationBase(io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase) InMemoryStorageFactory(io.pravega.segmentstore.storage.mocks.InMemoryStorageFactory) Duration(java.time.Duration) CachePolicy(io.pravega.segmentstore.server.CachePolicy) Operation(io.pravega.segmentstore.server.logs.operations.Operation) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ServiceListeners(io.pravega.segmentstore.server.ServiceListeners) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) Predicate(java.util.function.Predicate) Collection(java.util.Collection) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) Streams(com.google.common.collect.Streams) DataLogWriterNotPrimaryException(io.pravega.segmentstore.storage.DataLogWriterNotPrimaryException) Collectors(java.util.stream.Collectors) SegmentMetadataComparer(io.pravega.segmentstore.server.SegmentMetadataComparer) ErrorInjector(io.pravega.test.common.ErrorInjector) List(java.util.List) ByteArraySegment(io.pravega.common.util.ByteArraySegment) StreamSegmentContainerMetadata(io.pravega.segmentstore.server.containers.StreamSegmentContainerMetadata) DirectMemoryCache(io.pravega.segmentstore.storage.cache.DirectMemoryCache) TestUtils(io.pravega.test.common.TestUtils) Queue(java.util.Queue) Futures(io.pravega.common.concurrent.Futures) CacheManager(io.pravega.segmentstore.server.CacheManager) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) StreamSegmentException(io.pravega.segmentstore.contracts.StreamSegmentException) Exceptions(io.pravega.common.Exceptions) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) EvictableMetadata(io.pravega.segmentstore.server.EvictableMetadata) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) ReadIndexConfig(io.pravega.segmentstore.server.reading.ReadIndexConfig) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Timeout(org.junit.rules.Timeout) DurableDataLogException(io.pravega.segmentstore.storage.DurableDataLogException) OperationComparer(io.pravega.segmentstore.server.logs.operations.OperationComparer) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) DataLogDisabledException(io.pravega.segmentstore.storage.DataLogDisabledException) Iterator(java.util.Iterator) IntentionalException(io.pravega.test.common.IntentionalException) lombok.val(lombok.val) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) OperationLog(io.pravega.segmentstore.server.OperationLog) IOException(java.io.IOException) Test(org.junit.Test) TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) Service(com.google.common.util.concurrent.Service) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) TimeUnit(java.util.concurrent.TimeUnit) AbstractMap(java.util.AbstractMap) Rule(org.junit.Rule) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) Data(lombok.Data) ReadIndex(io.pravega.segmentstore.server.ReadIndex) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) Assert(org.junit.Assert) Collections(java.util.Collections) CompositeArrayView(io.pravega.common.util.CompositeArrayView) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) InputStream(java.io.InputStream) DirectMemoryCache(io.pravega.segmentstore.storage.cache.DirectMemoryCache) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) ArrayList(java.util.ArrayList) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) Cleanup(lombok.Cleanup) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) EvictableMetadata(io.pravega.segmentstore.server.EvictableMetadata) CacheManager(io.pravega.segmentstore.server.CacheManager) lombok.val(lombok.val) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) UpdateableSegmentMetadata(io.pravega.segmentstore.server.UpdateableSegmentMetadata) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) Storage(io.pravega.segmentstore.storage.Storage) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) Test(org.junit.Test)

Example 7 with ContainerReadIndex

use of io.pravega.segmentstore.server.reading.ContainerReadIndex in project pravega by pravega.

the class DurableLogTests method performPostFailureRecoveryChecks.

private void performPostFailureRecoveryChecks(ContainerSetup setup, int streamSegmentCount, List<OperationWithCompletion> completionFutures) {
    val recoveredMetadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, recoveredMetadata, setup.storage, setup.cacheManager, executorService());
    @Cleanup DurableLog recoveredLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), recoveredMetadata, setup.dataLogFactory, readIndex, executorService());
    recoveredLog.startAsync().awaitRunning();
    // Fetch recovered operations and skip over the segment creation ones.
    List<Operation> recoveredOperations = readUpToSequenceNumber(recoveredLog, recoveredMetadata.getOperationSequenceNumber());
    recoveredOperations = recoveredOperations.subList(1 + streamSegmentCount, recoveredOperations.size());
    val successfulOperationCount = completionFutures.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).count();
    // We expect that we recover the exact set of operations that were acked. However in the process of shutting down
    // we may have failed some operations that have been successfully written to DurableDataLog (i.e., persisted, but
    // the connection failed so we never got an ack). Since we cannot determine which of the "failed" operations should
    // have been successful, we can only check the ones that were.
    AssertExtensions.assertGreaterThanOrEqual("Fewer operations were recovered than were acked.", successfulOperationCount, recoveredOperations.size());
    val operationsToCheck = completionFutures.subList(0, recoveredOperations.size()).stream().map(oc -> oc.operation).collect(Collectors.toList());
    assertRecoveredOperationsMatch(operationsToCheck, recoveredOperations);
    // Stop the services.
    recoveredLog.stopAsync().awaitTerminated();
}
Also used : lombok.val(lombok.val) Storage(io.pravega.segmentstore.storage.Storage) StreamSegmentInformation(io.pravega.segmentstore.contracts.StreamSegmentInformation) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) OperationPriority(io.pravega.segmentstore.server.logs.operations.OperationPriority) SneakyThrows(lombok.SneakyThrows) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) AssertExtensions(io.pravega.test.common.AssertExtensions) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) TimeoutException(java.util.concurrent.TimeoutException) Cleanup(lombok.Cleanup) LogAddress(io.pravega.segmentstore.storage.LogAddress) UpdateableSegmentMetadata(io.pravega.segmentstore.server.UpdateableSegmentMetadata) DataLogNotAvailableException(io.pravega.segmentstore.storage.DataLogNotAvailableException) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CheckpointOperationBase(io.pravega.segmentstore.server.logs.operations.CheckpointOperationBase) InMemoryStorageFactory(io.pravega.segmentstore.storage.mocks.InMemoryStorageFactory) Duration(java.time.Duration) CachePolicy(io.pravega.segmentstore.server.CachePolicy) Operation(io.pravega.segmentstore.server.logs.operations.Operation) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ServiceListeners(io.pravega.segmentstore.server.ServiceListeners) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) Predicate(java.util.function.Predicate) Collection(java.util.Collection) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) Streams(com.google.common.collect.Streams) DataLogWriterNotPrimaryException(io.pravega.segmentstore.storage.DataLogWriterNotPrimaryException) Collectors(java.util.stream.Collectors) SegmentMetadataComparer(io.pravega.segmentstore.server.SegmentMetadataComparer) ErrorInjector(io.pravega.test.common.ErrorInjector) List(java.util.List) ByteArraySegment(io.pravega.common.util.ByteArraySegment) StreamSegmentContainerMetadata(io.pravega.segmentstore.server.containers.StreamSegmentContainerMetadata) DirectMemoryCache(io.pravega.segmentstore.storage.cache.DirectMemoryCache) TestUtils(io.pravega.test.common.TestUtils) Queue(java.util.Queue) Futures(io.pravega.common.concurrent.Futures) CacheManager(io.pravega.segmentstore.server.CacheManager) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) StreamSegmentException(io.pravega.segmentstore.contracts.StreamSegmentException) Exceptions(io.pravega.common.Exceptions) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) EvictableMetadata(io.pravega.segmentstore.server.EvictableMetadata) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) ReadIndexConfig(io.pravega.segmentstore.server.reading.ReadIndexConfig) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Timeout(org.junit.rules.Timeout) DurableDataLogException(io.pravega.segmentstore.storage.DurableDataLogException) OperationComparer(io.pravega.segmentstore.server.logs.operations.OperationComparer) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) DataLogDisabledException(io.pravega.segmentstore.storage.DataLogDisabledException) Iterator(java.util.Iterator) IntentionalException(io.pravega.test.common.IntentionalException) lombok.val(lombok.val) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) OperationLog(io.pravega.segmentstore.server.OperationLog) IOException(java.io.IOException) Test(org.junit.Test) TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) Service(com.google.common.util.concurrent.Service) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) TimeUnit(java.util.concurrent.TimeUnit) AbstractMap(java.util.AbstractMap) Rule(org.junit.Rule) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) Data(lombok.Data) ReadIndex(io.pravega.segmentstore.server.ReadIndex) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) Assert(org.junit.Assert) Collections(java.util.Collections) CompositeArrayView(io.pravega.common.util.CompositeArrayView) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) InputStream(java.io.InputStream) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) Cleanup(lombok.Cleanup) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex)

Example 8 with ContainerReadIndex

use of io.pravega.segmentstore.server.reading.ContainerReadIndex in project pravega by pravega.

the class DurableLogTests method testTruncateWithRecovery.

/**
 * Tests the truncate() method while performing recovery.
 */
@Test
public void testTruncateWithRecovery() {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    // Setup a DurableLog and start it.
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    AtomicReference<Boolean> truncationOccurred = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
    @Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
    @Cleanup ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, storage, cacheManager, executorService());
    Set<Long> streamSegmentIds;
    List<OperationWithCompletion> completionFutures;
    List<Operation> originalOperations;
    // First DurableLog. We use this for generating data.
    try (DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Generate some test data (we need to do this after we started the DurableLog because in the process of
        // recovery, it wipes away all existing metadata).
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, durableLog);
        List<Operation> queuedOperations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        completionFutures = processOperations(queuedOperations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        // Get a list of all the operations, before any truncation.
        originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Truncate up to each MetadataCheckpointOperation and:
    // * If the DataLog was truncated:
    // ** Shut down DurableLog, re-start it (recovery) and verify the operations are as they should.
    // At the end, verify all operations and all entries in the DataLog were truncated.
    DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService());
    try {
        durableLog.startAsync().awaitRunning();
        dataLog.get().setTruncateCallback(seqNo -> truncationOccurred.set(true));
        for (int i = 0; i < originalOperations.size(); i++) {
            Operation currentOperation = originalOperations.get(i);
            if (!(currentOperation instanceof MetadataCheckpointOperation)) {
                // We can only truncate on MetadataCheckpointOperations.
                continue;
            }
            truncationOccurred.set(false);
            durableLog.truncate(currentOperation.getSequenceNumber(), TIMEOUT).join();
            if (truncationOccurred.get()) {
                // Close current DurableLog and start a brand new one, forcing recovery.
                durableLog.close();
                durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService());
                durableLog.startAsync().awaitRunning();
                dataLog.get().setTruncateCallback(seqNo -> truncationOccurred.set(true));
                // Verify all operations up to, and including this one have been removed.
                Queue<Operation> reader = durableLog.read(2, TIMEOUT).join();
                Assert.assertFalse("Not expecting an empty log after truncating an operation (a MetadataCheckpoint must always exist).", reader.isEmpty());
                verifyFirstItemIsMetadataCheckpoint(reader.iterator());
                if (i < originalOperations.size() - 1) {
                    Operation firstOp = reader.poll();
                    OperationComparer.DEFAULT.assertEquals(String.format("Unexpected first operation after truncating SeqNo %d.", currentOperation.getSequenceNumber()), originalOperations.get(i + 1), firstOp);
                }
            }
        }
    } finally {
        // This closes whatever current instance this variable refers to, not necessarily the first one.
        durableLog.close();
    }
}
Also used : DirectMemoryCache(io.pravega.segmentstore.storage.cache.DirectMemoryCache) TestDurableDataLog(io.pravega.segmentstore.server.TestDurableDataLog) UpdateableContainerMetadata(io.pravega.segmentstore.server.UpdateableContainerMetadata) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) Cleanup(lombok.Cleanup) CacheManager(io.pravega.segmentstore.server.CacheManager) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) AtomicReference(java.util.concurrent.atomic.AtomicReference) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) Storage(io.pravega.segmentstore.storage.Storage) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) Test(org.junit.Test)

Example 9 with ContainerReadIndex

use of io.pravega.segmentstore.server.reading.ContainerReadIndex in project pravega by pravega.

the class DurableLogTests method testRecoveryWithMetadataCleanup.

/**
 * Tests the following recovery scenario:
 * 1. A Segment is created and recorded in the metadata with some optional operations executing on it.
 * 2. The segment is evicted from the metadata.
 * 3. The segment is reactivated (with a new metadata mapping) - possibly due to an append. No truncation since #2.
 * 4. Recovery.
 */
@Test
public void testRecoveryWithMetadataCleanup() throws Exception {
    final long truncatedSeqNo = Integer.MAX_VALUE;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    long segmentId;
    // First DurableLog. We use this for generating data.
    val metadata1 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
    @Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
    SegmentProperties originalSegmentInfo;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create the segment.
        val segmentIds = createStreamSegmentsWithOperations(1, durableLog);
        segmentId = segmentIds.stream().findFirst().orElse(-1L);
        // Evict the segment.
        val sm1 = metadata1.getStreamSegmentMetadata(segmentId);
        originalSegmentInfo = sm1.getSnapshot();
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata1.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata1.cleanup(Collections.singleton(sm1), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // Map the segment again.
        val reMapOp = new StreamSegmentMapOperation(originalSegmentInfo);
        reMapOp.setStreamSegmentId(segmentId);
        durableLog.add(reMapOp, OperationPriority.Normal, TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #1. This should work well.
    val metadata2 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Get segment info
        val recoveredSegmentInfo = metadata1.getStreamSegmentMetadata(segmentId).getSnapshot();
        Assert.assertEquals("Unexpected length from recovered segment.", originalSegmentInfo.getLength(), recoveredSegmentInfo.getLength());
        // Now evict the segment again ...
        val sm = metadata2.getStreamSegmentMetadata(segmentId);
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata2.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata2.cleanup(Collections.singleton(sm), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // ... and re-map it with a new Id. This is a perfectly valid operation, and we can't prevent it.
        durableLog.add(new StreamSegmentMapOperation(originalSegmentInfo), OperationPriority.Normal, TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #2. This should fail due to the same segment mapped multiple times with different ids.
    val metadata3 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata3, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata3, dataLogFactory, readIndex, executorService())) {
        AssertExtensions.assertThrows("Recovery did not fail with the expected exception in case of multi-mapping", () -> durableLog.startAsync().awaitRunning(), ex -> ex instanceof IllegalStateException && ex.getCause() instanceof DataCorruptionException && ex.getCause().getCause() instanceof MetadataUpdateException);
    }
}
Also used : lombok.val(lombok.val) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) DirectMemoryCache(io.pravega.segmentstore.storage.cache.DirectMemoryCache) MetadataBuilder(io.pravega.segmentstore.server.MetadataBuilder) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) ReadIndex(io.pravega.segmentstore.server.ReadIndex) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) InMemoryDurableDataLogFactory(io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory) Cleanup(lombok.Cleanup) ContainerReadIndex(io.pravega.segmentstore.server.reading.ContainerReadIndex) StreamSegmentContainerMetadata(io.pravega.segmentstore.server.containers.StreamSegmentContainerMetadata) Storage(io.pravega.segmentstore.storage.Storage) CacheStorage(io.pravega.segmentstore.storage.cache.CacheStorage) CacheManager(io.pravega.segmentstore.server.CacheManager) TestDurableDataLogFactory(io.pravega.segmentstore.server.TestDurableDataLogFactory) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) Test(org.junit.Test)

Aggregations

CacheManager (io.pravega.segmentstore.server.CacheManager)9 MetadataBuilder (io.pravega.segmentstore.server.MetadataBuilder)9 TestDurableDataLogFactory (io.pravega.segmentstore.server.TestDurableDataLogFactory)9 StreamSegmentMapOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation)9 ContainerReadIndex (io.pravega.segmentstore.server.reading.ContainerReadIndex)9 Storage (io.pravega.segmentstore.storage.Storage)9 CacheStorage (io.pravega.segmentstore.storage.cache.CacheStorage)9 DirectMemoryCache (io.pravega.segmentstore.storage.cache.DirectMemoryCache)9 InMemoryDurableDataLogFactory (io.pravega.segmentstore.storage.mocks.InMemoryDurableDataLogFactory)9 Cleanup (lombok.Cleanup)9 Test (org.junit.Test)9 ReadIndex (io.pravega.segmentstore.server.ReadIndex)8 CachedStreamSegmentAppendOperation (io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation)8 DeleteSegmentOperation (io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation)8 MergeSegmentOperation (io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation)8 MetadataCheckpointOperation (io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation)8 Operation (io.pravega.segmentstore.server.logs.operations.Operation)8 StorageMetadataCheckpointOperation (io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation)8 StorageOperation (io.pravega.segmentstore.server.logs.operations.StorageOperation)8 StreamSegmentAppendOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation)8