Examples with SegmentMetadata io.pravega.segmentstore.server.SegmentMetadata used on opensource projects

Example 46 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testCleanup.

/**
 * Tests the ability to evict Segment Metadatas that are not in use anymore.
 * 1. Creates a number of segments.
 * 2. Increases the truncated SeqNo in the metadata gradually and at each step verifies that the correct segments were evicted.
 * 3. Expires all segments and verifies they are all evicted.
 */
@Test
public void testCleanup() {
    // Expire each Segment at a different stage.
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    // Create a number of segments.
    // Each segment has a 'LastUsed' set in incremental order.
    final ArrayList<Long> segments = new ArrayList<>();
    populateSegmentsForEviction(segments, m);
    long maxLastUsed = 1;
    for (Long segmentId : segments) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segmentId);
        segmentMetadata.setLastUsed(maxLastUsed++);
    }
    final Map<Long, UpdateableSegmentMetadata> segmentMetadatas = segments.stream().collect(Collectors.toMap(id -> id, m::getStreamSegmentMetadata));
    // Truncate everything and expire all segments.
    m.removeTruncationMarkers(maxLastUsed);
    Collection<SegmentMetadata> evictionCandidates = m.getEvictionCandidates(maxLastUsed, Integer.MAX_VALUE);
    // Pick 3 segments and touch them. Then verify all but the 3 involved Segments are evicted.
    final long touchedSeqNo = maxLastUsed + 10;
    final ArrayList<Long> touchedSegments = new ArrayList<>();
    val iterator = segments.iterator();
    touchedSegments.add(iterator.next());
    touchedSegments.add(iterator.next());
    touchedSegments.add(iterator.next());
    segmentMetadatas.get(touchedSegments.get(0)).setLastUsed(touchedSeqNo);
    segmentMetadatas.get(touchedSegments.get(1)).setLastUsed(touchedSeqNo);
    segmentMetadatas.get(touchedSegments.get(2)).setLastUsed(touchedSeqNo);
    // Attempt to cleanup the eviction candidates, and even throw in a new truncation (to verify that alone won't trigger the cleanup).
    m.removeTruncationMarkers(touchedSeqNo + 1);
    Collection<SegmentMetadata> evictedSegments = m.cleanup(evictionCandidates, maxLastUsed);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    // Check that we evicted all eligible segments, and kept the 'touched' ones still.
    Assert.assertEquals("Unexpected number of segments were evicted (first-cleanup).", segments.size() - touchedSegments.size(), evictedSegments.size());
    for (long segmentId : touchedSegments) {
        SegmentMetadata sm = m.getStreamSegmentMetadata(segmentId);
        Assert.assertNotNull("Candidate segment that was touched was still evicted (lookup by id)", sm);
        Assert.assertEquals("Candidate segment that was touched was still evicted (lookup by name).", segmentId, m.getStreamSegmentId(sm.getName(), false));
        Assert.assertTrue("Non-evicted segment was marked as inactive.", sm.isActive());
    }
    // Now expire the remaining segments and verify.
    evictionCandidates = m.getEvictionCandidates(touchedSeqNo + 1, Integer.MAX_VALUE);
    evictedSegments = m.cleanup(evictionCandidates, touchedSeqNo + 1);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    Assert.assertEquals("Unexpected number of segments were evicted (second-cleanup).", touchedSegments.size(), evictedSegments.size());
    for (long segmentId : segments) {
        Assert.assertNull("Candidate segment was not evicted (lookup by id)", m.getStreamSegmentMetadata(segmentId));
    }
}

Example 47 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testGetEvictionCandidatesCapped.

/**
 * Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction when
 * there is an upper limit on how many such segments can be evicted at once.
 */
@Test
public void testGetEvictionCandidatesCapped() {
    final int maxEvictionCount = SEGMENT_COUNT / 10;
    final ArrayList<Long> segments = new ArrayList<>();
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        long segmentId = SEGMENT_COUNT - segments.size();
        m.mapStreamSegmentId(getName(segmentId), segmentId);
        segments.add(segmentId);
    }
    for (int i = 0; i < segments.size(); i++) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
        segmentMetadata.setLastUsed(i);
        m.removeTruncationMarkers(i + 1);
    }
    // Verify that not-yet-truncated operations will not be selected for truncation.
    Collection<SegmentMetadata> evictionCandidates;
    // capped, only the oldest-used segments are returned, in order.
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        int requestedCount = i + 1;
        evictionCandidates = m.getEvictionCandidates(requestedCount, maxEvictionCount);
        int expectedCount = Math.min(maxEvictionCount, requestedCount);
        Assert.assertEquals("Unexpected number of segments eligible for eviction.", expectedCount, evictionCandidates.size());
        if (requestedCount <= maxEvictionCount) {
            int expectedSegmentIndex = expectedCount - 1;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when less than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex--;
            }
        } else {
            // We were capped - make sure only the oldest-used segments are returned, in order.
            int expectedSegmentIndex = 0;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when more than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex++;
            }
        }
    }
}

Example 48 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class SegmentAggregator method initialize.

// endregion
// region Operations
/**
 * Initializes the SegmentAggregator by pulling information from the given Storage.
 *
 * @param timeout  Timeout for the operation.
 * @return A CompletableFuture that, when completed, will indicate that the operation finished successfully. If any
 * errors occurred during the operation, the Future will be completed with the appropriate exception.
 */
CompletableFuture<Void> initialize(Duration timeout) {
    Exceptions.checkNotClosed(isClosed(), this);
    Preconditions.checkState(this.state.get() == AggregatorState.NotInitialized, "SegmentAggregator has already been initialized.");
    assert this.handle.get() == null : "non-null handle but state == " + this.state.get();
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "initialize");
    if (this.metadata.isDeleted()) {
        // Segment is dead on arrival. Delete it from Storage (if it exists) and do not bother to do anything else with it).
        // This is a rather uncommon case, but it can happen in one of two cases: 1) the segment has been deleted
        // immediately after creation or 2) after a container recovery.
        log.info("{}: Segment '{}' is marked as Deleted in Metadata. Attempting Storage delete.", this.traceObjectId, this.metadata.getName());
        return Futures.exceptionallyExpecting(this.storage.openWrite(this.metadata.getName()).thenComposeAsync(handle -> this.storage.delete(handle, timeout), this.executor), ex -> ex instanceof StreamSegmentNotExistsException, // It's OK if already deleted.
        null).thenRun(() -> {
            updateMetadataPostDeletion(this.metadata);
            log.info("{}: Segment '{}' is marked as Deleted in Metadata and has been deleted from Storage. Ignoring all further operations on it.", this.traceObjectId, this.metadata.getName());
            setState(AggregatorState.Writing);
            LoggerHelpers.traceLeave(log, this.traceObjectId, "initialize", traceId);
        });
    }
    // Segment not deleted.
    return openWrite(this.metadata.getName(), this.handle, timeout).thenAcceptAsync(segmentInfo -> {
        // Check & Update StorageLength in metadata.
        if (this.metadata.getStorageLength() != segmentInfo.getLength()) {
            if (this.metadata.getStorageLength() >= 0) {
                // Only log warning if the StorageLength has actually been initialized, but is different.
                log.info("{}: SegmentMetadata has a StorageLength ({}) that is different than the actual one ({}) - updating metadata.", this.traceObjectId, this.metadata.getStorageLength(), segmentInfo.getLength());
            }
            // It is very important to keep this value up-to-date and correct.
            this.metadata.setStorageLength(segmentInfo.getLength());
        }
        // Check if the Storage segment is sealed, but it's not in metadata (this is 100% indicative of some data corruption happening).
        if (segmentInfo.isSealed()) {
            if (!this.metadata.isSealed()) {
                throw new CompletionException(new DataCorruptionException(String.format("Segment '%s' is sealed in Storage but not in the metadata.", this.metadata.getName())));
            }
            if (!this.metadata.isSealedInStorage()) {
                this.metadata.markSealedInStorage();
                log.info("{}: Segment is sealed in Storage but metadata does not reflect that - updating metadata.", this.traceObjectId);
            }
        }
        log.info("{}: Initialized. StorageLength = {}, Sealed = {}.", this.traceObjectId, segmentInfo.getLength(), segmentInfo.isSealed());
        LoggerHelpers.traceLeave(log, this.traceObjectId, "initialize", traceId);
        setState(AggregatorState.Writing);
    }, this.executor).exceptionally(ex -> {
        ex = Exceptions.unwrap(ex);
        if (ex instanceof StreamSegmentNotExistsException) {
            // Segment does not exist in Storage. There are two possibilities here:
            if (this.metadata.getStorageLength() == 0 && !this.metadata.isDeletedInStorage()) {
                // Segment has never been created because there was nothing to write to it. As long as we know
                // its expected length is zero, this is a valid case.
                this.handle.set(null);
                log.info("{}: Initialized. Segment does not exist in Storage but Metadata indicates it should be empty.", this.traceObjectId);
                if (this.metadata.isSealed() && this.metadata.getLength() == 0) {
                    // Truly an empty segment that is sealed; mark it as such in Storage.
                    this.metadata.markSealedInStorage();
                    log.info("{}: Segment does not exist in Storage, but Metadata indicates it is empty and sealed - marking as sealed in storage.", this.traceObjectId);
                }
            } else {
                // Segment does not exist anymore. This is a real possibility during recovery, in the following cases:
                // * We already processed a Segment Deletion but did not have a chance to checkpoint metadata
                // * We processed a MergeSegmentOperation but did not have a chance to ack/truncate the DataSource
                // Update metadata, just in case it is not already updated.
                updateMetadataPostDeletion(this.metadata);
                log.info("{}: Segment '{}' does not exist in Storage. Ignoring all further operations on it.", this.traceObjectId, this.metadata.getName());
            }
            setState(AggregatorState.Writing);
            LoggerHelpers.traceLeave(log, this.traceObjectId, "initialize", traceId);
        } else {
            // Other kind of error - re-throw.
            throw new CompletionException(ex);
        }
        return null;
    });
}

Example 49 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class DurableLogTests method testRecoveryWithIncrementalCheckpoints.

/**
 * Tests the DurableLog recovery process when there are multiple {@link MetadataCheckpointOperation}s added, with each
 * such checkpoint including information about evicted segments or segments which had their storage state modified.
 */
@Test
public void testRecoveryWithIncrementalCheckpoints() throws Exception {
    final int streamSegmentCount = 50;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    // First DurableLog. We use this for generating data.
    val metadata1 = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
    @Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
    List<Long> deletedIds;
    Set<Long> evictIds;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create some segments.
        val segmentIds = new ArrayList<>(createStreamSegmentsWithOperations(streamSegmentCount, durableLog));
        deletedIds = segmentIds.subList(0, 5);
        val mergedFromIds = segmentIds.subList(5, 10);
        // Must be same length as mergeFrom
        val mergedToIds = segmentIds.subList(10, 15);
        evictIds = new HashSet<>(segmentIds.subList(15, 20));
        val changeStorageStateIds = segmentIds.subList(20, segmentIds.size() - 5);
        // Append something to each segment.
        for (val segmentId : segmentIds) {
            if (!evictIds.contains(segmentId)) {
                durableLog.add(new StreamSegmentAppendOperation(segmentId, generateAppendData((int) (long) segmentId), null), OperationPriority.Normal, TIMEOUT).join();
            }
        }
        // Checkpoint 1.
        durableLog.checkpoint(TIMEOUT).join();
        // Delete some segments.
        for (val segmentId : deletedIds) {
            durableLog.add(new DeleteSegmentOperation(segmentId), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 2.
        durableLog.checkpoint(TIMEOUT).join();
        // Merge some segments.
        for (int i = 0; i < mergedFromIds.size(); i++) {
            durableLog.add(new StreamSegmentSealOperation(mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
            durableLog.add(new MergeSegmentOperation(mergedToIds.get(i), mergedFromIds.get(i)), OperationPriority.Normal, TIMEOUT).join();
        }
        // Checkpoint 3.
        durableLog.checkpoint(TIMEOUT).join();
        // Evict some segments.
        val evictableContainerMetadata = (EvictableMetadata) metadata1;
        metadata1.removeTruncationMarkers(metadata1.getOperationSequenceNumber());
        val toEvict = evictableContainerMetadata.getEvictionCandidates(Integer.MAX_VALUE, segmentIds.size()).stream().filter(m -> evictIds.contains(m.getId())).collect(Collectors.toList());
        val evicted = evictableContainerMetadata.cleanup(toEvict, Integer.MAX_VALUE);
        AssertExtensions.assertContainsSameElements("", evictIds, evicted.stream().map(SegmentMetadata::getId).collect(Collectors.toList()));
        // Checkpoint 4.
        durableLog.checkpoint(TIMEOUT).join();
        // Update storage state for some segments.
        for (val segmentId : changeStorageStateIds) {
            val sm = metadata1.getStreamSegmentMetadata(segmentId);
            if (segmentId % 3 == 0) {
                sm.setStorageLength(sm.getLength());
            }
            if (segmentId % 4 == 0) {
                sm.markSealed();
                sm.markSealedInStorage();
            }
            if (segmentId % 5 == 0) {
                sm.markDeleted();
                sm.markDeletedInStorage();
            }
        }
        // Checkpoint 5.
        durableLog.checkpoint(TIMEOUT).join();
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Second DurableLog. We use this for recovery.
    val metadata2 = new MetadataBuilder(CONTAINER_ID).build();
    try (ContainerReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Validate metadata matches.
        val expectedSegmentIds = metadata1.getAllStreamSegmentIds();
        val actualSegmentIds = metadata2.getAllStreamSegmentIds();
        AssertExtensions.assertContainsSameElements("Unexpected set of recovered segments. Only Active segments expected to have been recovered.", expectedSegmentIds, actualSegmentIds);
        val expectedSegments = expectedSegmentIds.stream().sorted().map(metadata1::getStreamSegmentMetadata).collect(Collectors.toList());
        val actualSegments = actualSegmentIds.stream().sorted().map(metadata2::getStreamSegmentMetadata).collect(Collectors.toList());
        for (int i = 0; i < expectedSegments.size(); i++) {
            val e = expectedSegments.get(i);
            val a = actualSegments.get(i);
            SegmentMetadataComparer.assertEquals("Recovered segment metadata mismatch", e, a);
        }
        // Validate read index is as it should. Here, we can only check if the read indices for evicted segments are
        // no longer loaded; we do more thorough checks in the ContainerReadIndexTests suite.
        Streams.concat(evictIds.stream(), deletedIds.stream()).forEach(segmentId -> Assert.assertNull("Not expecting a read index for an evicted or deleted segment.", readIndex.getIndex(segmentId)));
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
}

Example 50 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class OperationLogTestBase method performMetadataChecks.

// endregion
// region Verification
void performMetadataChecks(Collection<Long> streamSegmentIds, Collection<Long> invalidStreamSegmentIds, Map<Long, Long> transactions, Collection<OperationWithCompletion> operations, ContainerMetadata metadata, boolean expectTransactionsMerged, boolean expectSegmentsSealed) {
    // Verify that transactions are merged
    for (long transactionId : transactions.keySet()) {
        SegmentMetadata transactionMetadata = metadata.getStreamSegmentMetadata(transactionId);
        if (invalidStreamSegmentIds.contains(transactionId)) {
            Assert.assertTrue("Unexpected data for a Transaction that was invalid.", transactionMetadata == null || transactionMetadata.getLength() == 0);
        } else {
            Assert.assertEquals("Unexpected Transaction seal state for Transaction " + transactionId, expectTransactionsMerged, transactionMetadata.isSealed());
            Assert.assertEquals("Unexpected Transaction merge state for Transaction " + transactionId, expectTransactionsMerged, transactionMetadata.isMerged());
        }
    }
    // Verify the end state of each stream segment (length, sealed).
    AbstractMap<Long, Integer> expectedLengths = getExpectedLengths(operations);
    for (long streamSegmentId : streamSegmentIds) {
        SegmentMetadata segmentMetadata = metadata.getStreamSegmentMetadata(streamSegmentId);
        if (invalidStreamSegmentIds.contains(streamSegmentId)) {
            Assert.assertTrue("Unexpected data for a StreamSegment that was invalid.", segmentMetadata == null || segmentMetadata.getLength() == 0);
        } else {
            Assert.assertEquals("Unexpected seal state for StreamSegment " + streamSegmentId, expectSegmentsSealed, segmentMetadata.isSealed());
            Assert.assertEquals("Unexpected length for StreamSegment " + streamSegmentId, (int) expectedLengths.getOrDefault(streamSegmentId, 0), segmentMetadata.getLength());
        }
    }
}