Examples with MetadataBuilder io.pravega.segmentstore.server.MetadataBuilder used on opensource projects

Example 16 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testEpoch.

/**
 * Tests Epoch-related operations.
 */
@Test
public void testEpoch() {
    final int epoch = 10;
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
    AssertExtensions.assertThrows("setContainerEpoch allowed updating the sequence number in non-recovery mode.", () -> m.setContainerEpoch(Integer.MAX_VALUE), ex -> ex instanceof IllegalStateException);
    // In recovery mode: setContainerEpoch should work.
    m.enterRecoveryMode();
    m.setContainerEpoch(epoch);
    Assert.assertEquals("Unexpected value from getContainerEpoch.", epoch, m.getContainerEpoch());
    AssertExtensions.assertThrows("setContainerEpoch allowed updating the epoch after the initial set.", () -> m.setContainerEpoch(11), ex -> ex instanceof IllegalStateException);
    Assert.assertEquals("Unexpected value from getContainerEpoch after rejected update.", epoch, m.getContainerEpoch());
    m.exitRecoveryMode();
    Assert.assertEquals("Unexpected value from getContainerEpoch after exit from recovery mode.", epoch, m.getContainerEpoch());
}

Example 17 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testValidTruncationPoints.

/**
 * Tests the ability to set and retrieve truncation points (truncation markers is tested separately).
 */
@Test
public void testValidTruncationPoints() {
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
    for (int i = 0; i < 100; i += 2) {
        m.setValidTruncationPoint(i);
    }
    for (int i = 0; i < 100; i++) {
        boolean expectedValid = i % 2 == 0;
        Assert.assertEquals("Unexpected result from isValidTruncationPoint.", expectedValid, m.isValidTruncationPoint(i));
    }
}

Example 18 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testGetEvictionCandidates.

/**
 * Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction.
 * 1. Creates a number of segment, and 1/4 of them have transactions.
 * 2. All transactions are set to expire at a particular time and the segments expire in two separate stages.
 * 3. Truncates repeatedly and at each step verifies that the correct segments were identified as candidates.
 * 4. "Expires" all transactions and verifies that all dependent segments (which are eligible) are also identified.
 * 5. "Expires" all segments and verifies they are all identified as candidates.
 */
@Test
public void testGetEvictionCandidates() {
    // Expire each segment at a different stage.
    final long firstStageExpiration = SEGMENT_COUNT;
    final long transactionExpiration = firstStageExpiration + SEGMENT_COUNT;
    final long finalExpiration = transactionExpiration + SEGMENT_COUNT;
    // Create a number of segments, out of which every 4th one has a transaction (25%).
    // Each segment has a 'LastKnownSequenceNumber' set in incremental order.
    final ArrayList<Long> segments = new ArrayList<>();
    final HashMap<Long, Long> transactions = new HashMap<>();
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    populateSegmentsForEviction(segments, transactions, m);
    for (int i = 0; i < segments.size(); i++) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
        if (segmentMetadata.isTransaction()) {
            // All transactions expire at once, in a second step.
            segmentMetadata.setLastUsed(transactionExpiration);
        } else if (i % 2 == 0) {
            // 1/2 of segments expire at the end.
            segmentMetadata.setLastUsed(finalExpiration);
        } else {
            // The rest of the segments expire in the first stage.
            segmentMetadata.setLastUsed(firstStageExpiration);
        }
    }
    // Add one segment that will be deleted. This should be evicted as soon as its LastUsed is before the truncation point.
    final long deletedSegmentId = segments.size();
    UpdateableSegmentMetadata deletedSegment = m.mapStreamSegmentId(getName(deletedSegmentId), deletedSegmentId);
    deletedSegment.markDeleted();
    deletedSegment.setLastUsed(firstStageExpiration);
    segments.add(deletedSegmentId);
    // Verify that not-yet-truncated operations will not be selected for truncation.
    val truncationPoints = Arrays.asList(0L, firstStageExpiration, transactionExpiration, finalExpiration, finalExpiration + 1);
    Collection<SegmentMetadata> evictionCandidates;
    for (long truncatedSeqNo : truncationPoints) {
        // Simulate a truncation.
        m.removeTruncationMarkers(truncatedSeqNo);
        // Try to evict everything.
        evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
        checkEvictedSegmentCandidates(evictionCandidates, transactions, m, finalExpiration + 1, truncatedSeqNo);
    }
    // Now we expire transactions.
    evictionCandidates = m.getEvictionCandidates(transactionExpiration + 1, Integer.MAX_VALUE);
    checkEvictedSegmentCandidates(evictionCandidates, transactions, m, transactionExpiration + 1, Long.MAX_VALUE);
    // Now we expire all segments.
    evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
    checkEvictedSegmentCandidates(evictionCandidates, transactions, m, finalExpiration + 1, Long.MAX_VALUE);
    // Check that, in the end, all segments in the metadata have been selected for eviction.
    Assert.assertEquals("Not all segments were evicted.", segments.size(), evictionCandidates.size());
}

Example 19 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testGetEvictionCandidatesCapped.

/**
 * Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction when
 * there is an upper limit on how many such segments can be evicted at once.
 */
@Test
public void testGetEvictionCandidatesCapped() {
    final int maxEvictionCount = SEGMENT_COUNT / 10;
    final ArrayList<Long> segments = new ArrayList<>();
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        long segmentId = SEGMENT_COUNT - segments.size();
        m.mapStreamSegmentId(getName(segmentId), segmentId);
        segments.add(segmentId);
    }
    for (int i = 0; i < segments.size(); i++) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
        segmentMetadata.setLastUsed(i);
        m.removeTruncationMarkers(i + 1);
    }
    // Verify that not-yet-truncated operations will not be selected for truncation.
    Collection<SegmentMetadata> evictionCandidates;
    // capped, only the oldest-used segments are returned, in order.
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        int requestedCount = i + 1;
        evictionCandidates = m.getEvictionCandidates(requestedCount, maxEvictionCount);
        int expectedCount = Math.min(maxEvictionCount, requestedCount);
        Assert.assertEquals("Unexpected number of segments eligible for eviction.", expectedCount, evictionCandidates.size());
        if (requestedCount <= maxEvictionCount) {
            int expectedSegmentIndex = expectedCount - 1;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when less than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex--;
            }
        } else {
            // We were capped - make sure only the oldest-used segments are returned, in order.
            int expectedSegmentIndex = 0;
            for (SegmentMetadata candidate : evictionCandidates) {
                Assert.assertEquals("Unexpected segment id chosen for eviction when more than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
                expectedSegmentIndex++;
            }
        }
    }
}

Example 20 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testCleanup.

/**
 * Tests the ability to evict Segment Metadatas that are not in use anymore.
 * 1. Creates a number of segment, and 1/4 of them have transactions.
 * 2. All transactions are set to expire at a particular time and the segments expire in two separate stages.
 * 3. Increases the truncated SeqNo in the metadata gradually and at each step verifies that the correct segments were evicted.
 * 4. Expires all transactions and verifies that all dependent segments (which are eligible) are also evicted.
 * 5. Expires all segments and verifies they are all evicted.
 */
@Test
public void testCleanup() {
    // Expire each Segment at a different stage.
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    // Create a number of segments, out of which every 4th one has a transaction (25%).
    // Each segment has a 'LastUsed' set in incremental order.
    final ArrayList<Long> segments = new ArrayList<>();
    final HashMap<Long, Long> transactions = new HashMap<>();
    populateSegmentsForEviction(segments, transactions, m);
    long maxLastUsed = 1;
    for (Long segmentId : segments) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segmentId);
        segmentMetadata.setLastUsed(maxLastUsed++);
    }
    final Map<Long, UpdateableSegmentMetadata> segmentMetadatas = segments.stream().collect(Collectors.toMap(id -> id, m::getStreamSegmentMetadata));
    // Truncate everything and expire all segments.
    m.removeTruncationMarkers(maxLastUsed);
    Collection<SegmentMetadata> evictionCandidates = m.getEvictionCandidates(maxLastUsed, Integer.MAX_VALUE);
    // Pick a Transaction and a non-related Segment and touch them. Then verify all but the three involved Segments are evicted.
    final long touchedSeqNo = maxLastUsed + 10;
    final ArrayList<Long> touchedSegments = new ArrayList<>();
    val iterator = transactions.entrySet().iterator();
    touchedSegments.add(iterator.next().getKey());
    val second = iterator.next();
    touchedSegments.add(second.getValue());
    segmentMetadatas.get(touchedSegments.get(0)).setLastUsed(touchedSeqNo);
    segmentMetadatas.get(touchedSegments.get(1)).setLastUsed(touchedSeqNo);
    // We add the Transaction's parent, but do not touch it.
    touchedSegments.add(second.getKey());
    // Attempt to cleanup the eviction candidates, and even throw in a new truncation (to verify that alone won't trigger the cleanup).
    m.removeTruncationMarkers(touchedSeqNo + 1);
    Collection<SegmentMetadata> evictedSegments = m.cleanup(evictionCandidates, maxLastUsed);
    verifyMetadataConsistency(m);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    // Check that we evicted all eligible segments, and kept the 'touched' ones still.
    Assert.assertEquals("Unexpected number of segments were evicted (first-cleanup).", segments.size() - touchedSegments.size(), evictedSegments.size());
    for (long segmentId : touchedSegments) {
        SegmentMetadata sm = m.getStreamSegmentMetadata(segmentId);
        Assert.assertNotNull("Candidate segment that was touched was still evicted (lookup by id)", sm);
        Assert.assertEquals("Candidate segment that was touched was still evicted (lookup by name).", segmentId, m.getStreamSegmentId(sm.getName(), false));
        Assert.assertTrue("Non-evicted segment was marked as inactive.", sm.isActive());
    }
    // Now expire the remaining segments and verify.
    evictionCandidates = m.getEvictionCandidates(touchedSeqNo + 1, Integer.MAX_VALUE);
    evictedSegments = m.cleanup(evictionCandidates, touchedSeqNo + 1);
    verifyMetadataConsistency(m);
    for (SegmentMetadata sm : evictedSegments) {
        Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
    }
    Assert.assertEquals("Unexpected number of segments were evicted (second-cleanup).", touchedSegments.size(), evictedSegments.size());
    for (long segmentId : segments) {
        Assert.assertNull("Candidate segment was not evicted (lookup by id)", m.getStreamSegmentMetadata(segmentId));
    }
}