Examples with UpdateableSegmentMetadata io.pravega.segmentstore.server.UpdateableSegmentMetadata used on opensource projects

Example 51 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testGetEvictionCandidatesCapped.

/**
 * Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction when
 * there is an upper limit on how many such segments can be evicted at once.
 */
@Test
public void testGetEvictionCandidatesCapped() {
    final int maxEvictionCount = SEGMENT_COUNT / 10;
    final ArrayList<Long> segments = new ArrayList<>();
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        long segmentId = SEGMENT_COUNT - segments.size();
        m.mapStreamSegmentId(getName(segmentId), segmentId);
        segments.add(segmentId);
    }
    for (int i = 0; i < segments.size(); i++) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
        segmentMetadata.setLastUsed(i);
        m.removeTruncationMarkers(i + 1);
    }
    // Verify that not-yet-truncated operations will not be selected for truncation.
    Collection<SegmentMetadata> evictionCandidates;
    // capped, only the oldest-used segments are returned, in order.
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        int requestedCount = i + 1;
        evictionCandidates = m.getEvictionCandidates(requestedCount, maxEvictionCount);
        int expectedCount = Math.min(maxEvictionCount, requestedCount);
        Assert.assertEquals("Unexpected number of segments eligible for eviction.", expectedCount, evictionCandidates.size());
        if (requestedCount <= maxEvictionCount) {
            int expectedSegmentIndex = expectedCount - 1;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when less than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex--;
            }
        } else {
            // We were capped - make sure only the oldest-used segments are returned, in order.
            int expectedSegmentIndex = 0;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when more than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex++;
            }
        }
    }
}

Example 52 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testCleanup.

/**
 * Tests the ability to evict Segment Metadatas that are not in use anymore.
 * 1. Creates a number of segment, and 1/4 of them have transactions.
 * 2. All transactions are set to expire at a particular time and the segments expire in two separate stages.
 * 3. Increases the truncated SeqNo in the metadata gradually and at each step verifies that the correct segments were evicted.
 * 4. Expires all transactions and verifies that all dependent segments (which are eligible) are also evicted.
 * 5. Expires all segments and verifies they are all evicted.
 */
@Test
public void testCleanup() {
    // Expire each Segment at a different stage.
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    // Create a number of segments, out of which every 4th one has a transaction (25%).
    // Each segment has a 'LastUsed' set in incremental order.
    final ArrayList<Long> segments = new ArrayList<>();
    final HashMap<Long, Long> transactions = new HashMap<>();
    populateSegmentsForEviction(segments, transactions, m);
    long maxLastUsed = 1;
    for (Long segmentId : segments) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segmentId);
        segmentMetadata.setLastUsed(maxLastUsed++);
    }
    final Map<Long, UpdateableSegmentMetadata> segmentMetadatas = segments.stream().collect(Collectors.toMap(id -> id, m::getStreamSegmentMetadata));
    // Truncate everything and expire all segments.
    m.removeTruncationMarkers(maxLastUsed);
    Collection<SegmentMetadata> evictionCandidates = m.getEvictionCandidates(maxLastUsed, Integer.MAX_VALUE);
    // Pick a Transaction and a non-related Segment and touch them. Then verify all but the three involved Segments are evicted.
    final long touchedSeqNo = maxLastUsed + 10;
    final ArrayList<Long> touchedSegments = new ArrayList<>();
    val iterator = transactions.entrySet().iterator();
    touchedSegments.add(iterator.next().getKey());
    val second = iterator.next();
    touchedSegments.add(second.getValue());
    segmentMetadatas.get(touchedSegments.get(0)).setLastUsed(touchedSeqNo);
    segmentMetadatas.get(touchedSegments.get(1)).setLastUsed(touchedSeqNo);
    // We add the Transaction's parent, but do not touch it.
    touchedSegments.add(second.getKey());
    // Attempt to cleanup the eviction candidates, and even throw in a new truncation (to verify that alone won't trigger the cleanup).
    m.removeTruncationMarkers(touchedSeqNo + 1);
    Collection<SegmentMetadata> evictedSegments = m.cleanup(evictionCandidates, maxLastUsed);
    verifyMetadataConsistency(m);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    // Check that we evicted all eligible segments, and kept the 'touched' ones still.
    Assert.assertEquals("Unexpected number of segments were evicted (first-cleanup).", segments.size() - touchedSegments.size(), evictedSegments.size());
    for (long segmentId : touchedSegments) {
        SegmentMetadata sm = m.getStreamSegmentMetadata(segmentId);
        Assert.assertNotNull("Candidate segment that was touched was still evicted (lookup by id)", sm);
        Assert.assertEquals("Candidate segment that was touched was still evicted (lookup by name).", segmentId, m.getStreamSegmentId(sm.getName(), false));
        Assert.assertTrue("Non-evicted segment was marked as inactive.", sm.isActive());
    }
    // Now expire the remaining segments and verify.
    evictionCandidates = m.getEvictionCandidates(touchedSeqNo + 1, Integer.MAX_VALUE);
    evictedSegments = m.cleanup(evictionCandidates, touchedSeqNo + 1);
    verifyMetadataConsistency(m);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    Assert.assertEquals("Unexpected number of segments were evicted (second-cleanup).", touchedSegments.size(), evictedSegments.size());
    for (long segmentId : segments) {
        Assert.assertNull("Candidate segment was not evicted (lookup by id)", m.getStreamSegmentMetadata(segmentId));
    }
}