use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.
the class StreamSegmentContainerMetadataTests method testEpoch.
/**
* Tests Epoch-related operations.
*/
@Test
public void testEpoch() {
final int epoch = 10;
final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
AssertExtensions.assertThrows("setContainerEpoch allowed updating the sequence number in non-recovery mode.", () -> m.setContainerEpoch(Integer.MAX_VALUE), ex -> ex instanceof IllegalStateException);
// In recovery mode: setContainerEpoch should work.
m.enterRecoveryMode();
m.setContainerEpoch(epoch);
Assert.assertEquals("Unexpected value from getContainerEpoch.", epoch, m.getContainerEpoch());
AssertExtensions.assertThrows("setContainerEpoch allowed updating the epoch after the initial set.", () -> m.setContainerEpoch(11), ex -> ex instanceof IllegalStateException);
Assert.assertEquals("Unexpected value from getContainerEpoch after rejected update.", epoch, m.getContainerEpoch());
m.exitRecoveryMode();
Assert.assertEquals("Unexpected value from getContainerEpoch after exit from recovery mode.", epoch, m.getContainerEpoch());
}
use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.
the class StreamSegmentContainerMetadataTests method testValidTruncationPoints.
/**
* Tests the ability to set and retrieve truncation points (truncation markers is tested separately).
*/
@Test
public void testValidTruncationPoints() {
final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
for (int i = 0; i < 100; i += 2) {
m.setValidTruncationPoint(i);
}
for (int i = 0; i < 100; i++) {
boolean expectedValid = i % 2 == 0;
Assert.assertEquals("Unexpected result from isValidTruncationPoint.", expectedValid, m.isValidTruncationPoint(i));
}
}
use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.
the class StreamSegmentContainerMetadataTests method testGetEvictionCandidates.
/**
* Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction.
* 1. Creates a number of segment, and 1/4 of them have transactions.
* 2. All transactions are set to expire at a particular time and the segments expire in two separate stages.
* 3. Truncates repeatedly and at each step verifies that the correct segments were identified as candidates.
* 4. "Expires" all transactions and verifies that all dependent segments (which are eligible) are also identified.
* 5. "Expires" all segments and verifies they are all identified as candidates.
*/
@Test
public void testGetEvictionCandidates() {
// Expire each segment at a different stage.
final long firstStageExpiration = SEGMENT_COUNT;
final long transactionExpiration = firstStageExpiration + SEGMENT_COUNT;
final long finalExpiration = transactionExpiration + SEGMENT_COUNT;
// Create a number of segments, out of which every 4th one has a transaction (25%).
// Each segment has a 'LastKnownSequenceNumber' set in incremental order.
final ArrayList<Long> segments = new ArrayList<>();
final HashMap<Long, Long> transactions = new HashMap<>();
final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
populateSegmentsForEviction(segments, transactions, m);
for (int i = 0; i < segments.size(); i++) {
UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
if (segmentMetadata.isTransaction()) {
// All transactions expire at once, in a second step.
segmentMetadata.setLastUsed(transactionExpiration);
} else if (i % 2 == 0) {
// 1/2 of segments expire at the end.
segmentMetadata.setLastUsed(finalExpiration);
} else {
// The rest of the segments expire in the first stage.
segmentMetadata.setLastUsed(firstStageExpiration);
}
}
// Add one segment that will be deleted. This should be evicted as soon as its LastUsed is before the truncation point.
final long deletedSegmentId = segments.size();
UpdateableSegmentMetadata deletedSegment = m.mapStreamSegmentId(getName(deletedSegmentId), deletedSegmentId);
deletedSegment.markDeleted();
deletedSegment.setLastUsed(firstStageExpiration);
segments.add(deletedSegmentId);
// Verify that not-yet-truncated operations will not be selected for truncation.
val truncationPoints = Arrays.asList(0L, firstStageExpiration, transactionExpiration, finalExpiration, finalExpiration + 1);
Collection<SegmentMetadata> evictionCandidates;
for (long truncatedSeqNo : truncationPoints) {
// Simulate a truncation.
m.removeTruncationMarkers(truncatedSeqNo);
// Try to evict everything.
evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
checkEvictedSegmentCandidates(evictionCandidates, transactions, m, finalExpiration + 1, truncatedSeqNo);
}
// Now we expire transactions.
evictionCandidates = m.getEvictionCandidates(transactionExpiration + 1, Integer.MAX_VALUE);
checkEvictedSegmentCandidates(evictionCandidates, transactions, m, transactionExpiration + 1, Long.MAX_VALUE);
// Now we expire all segments.
evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
checkEvictedSegmentCandidates(evictionCandidates, transactions, m, finalExpiration + 1, Long.MAX_VALUE);
// Check that, in the end, all segments in the metadata have been selected for eviction.
Assert.assertEquals("Not all segments were evicted.", segments.size(), evictionCandidates.size());
}
use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.
the class StreamSegmentContainerMetadataTests method testGetEvictionCandidatesCapped.
/**
* Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction when
* there is an upper limit on how many such segments can be evicted at once.
*/
@Test
public void testGetEvictionCandidatesCapped() {
final int maxEvictionCount = SEGMENT_COUNT / 10;
final ArrayList<Long> segments = new ArrayList<>();
final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
for (int i = 0; i < SEGMENT_COUNT; i++) {
long segmentId = SEGMENT_COUNT - segments.size();
m.mapStreamSegmentId(getName(segmentId), segmentId);
segments.add(segmentId);
}
for (int i = 0; i < segments.size(); i++) {
UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
segmentMetadata.setLastUsed(i);
m.removeTruncationMarkers(i + 1);
}
// Verify that not-yet-truncated operations will not be selected for truncation.
Collection<SegmentMetadata> evictionCandidates;
// capped, only the oldest-used segments are returned, in order.
for (int i = 0; i < SEGMENT_COUNT; i++) {
int requestedCount = i + 1;
evictionCandidates = m.getEvictionCandidates(requestedCount, maxEvictionCount);
int expectedCount = Math.min(maxEvictionCount, requestedCount);
Assert.assertEquals("Unexpected number of segments eligible for eviction.", expectedCount, evictionCandidates.size());
if (requestedCount <= maxEvictionCount) {
int expectedSegmentIndex = expectedCount - 1;
for (SegmentMetadata candidate : evictionCandidates) {
Assert.assertEquals("Unexpected segment id chosen for eviction when less than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
expectedSegmentIndex--;
}
} else {
// We were capped - make sure only the oldest-used segments are returned, in order.
int expectedSegmentIndex = 0;
for (SegmentMetadata candidate : evictionCandidates) {
Assert.assertEquals("Unexpected segment id chosen for eviction when more than Max.", (long) segments.get(expectedSegmentIndex), candidate.getId());
expectedSegmentIndex++;
}
}
}
}
use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.
the class StreamSegmentContainerMetadataTests method testCleanup.
/**
* Tests the ability to evict Segment Metadatas that are not in use anymore.
* 1. Creates a number of segment, and 1/4 of them have transactions.
* 2. All transactions are set to expire at a particular time and the segments expire in two separate stages.
* 3. Increases the truncated SeqNo in the metadata gradually and at each step verifies that the correct segments were evicted.
* 4. Expires all transactions and verifies that all dependent segments (which are eligible) are also evicted.
* 5. Expires all segments and verifies they are all evicted.
*/
@Test
public void testCleanup() {
// Expire each Segment at a different stage.
final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
// Create a number of segments, out of which every 4th one has a transaction (25%).
// Each segment has a 'LastUsed' set in incremental order.
final ArrayList<Long> segments = new ArrayList<>();
final HashMap<Long, Long> transactions = new HashMap<>();
populateSegmentsForEviction(segments, transactions, m);
long maxLastUsed = 1;
for (Long segmentId : segments) {
UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segmentId);
segmentMetadata.setLastUsed(maxLastUsed++);
}
final Map<Long, UpdateableSegmentMetadata> segmentMetadatas = segments.stream().collect(Collectors.toMap(id -> id, m::getStreamSegmentMetadata));
// Truncate everything and expire all segments.
m.removeTruncationMarkers(maxLastUsed);
Collection<SegmentMetadata> evictionCandidates = m.getEvictionCandidates(maxLastUsed, Integer.MAX_VALUE);
// Pick a Transaction and a non-related Segment and touch them. Then verify all but the three involved Segments are evicted.
final long touchedSeqNo = maxLastUsed + 10;
final ArrayList<Long> touchedSegments = new ArrayList<>();
val iterator = transactions.entrySet().iterator();
touchedSegments.add(iterator.next().getKey());
val second = iterator.next();
touchedSegments.add(second.getValue());
segmentMetadatas.get(touchedSegments.get(0)).setLastUsed(touchedSeqNo);
segmentMetadatas.get(touchedSegments.get(1)).setLastUsed(touchedSeqNo);
// We add the Transaction's parent, but do not touch it.
touchedSegments.add(second.getKey());
// Attempt to cleanup the eviction candidates, and even throw in a new truncation (to verify that alone won't trigger the cleanup).
m.removeTruncationMarkers(touchedSeqNo + 1);
Collection<SegmentMetadata> evictedSegments = m.cleanup(evictionCandidates, maxLastUsed);
verifyMetadataConsistency(m);
for (SegmentMetadata sm : evictedSegments) {
Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
}
// Check that we evicted all eligible segments, and kept the 'touched' ones still.
Assert.assertEquals("Unexpected number of segments were evicted (first-cleanup).", segments.size() - touchedSegments.size(), evictedSegments.size());
for (long segmentId : touchedSegments) {
SegmentMetadata sm = m.getStreamSegmentMetadata(segmentId);
Assert.assertNotNull("Candidate segment that was touched was still evicted (lookup by id)", sm);
Assert.assertEquals("Candidate segment that was touched was still evicted (lookup by name).", segmentId, m.getStreamSegmentId(sm.getName(), false));
Assert.assertTrue("Non-evicted segment was marked as inactive.", sm.isActive());
}
// Now expire the remaining segments and verify.
evictionCandidates = m.getEvictionCandidates(touchedSeqNo + 1, Integer.MAX_VALUE);
evictedSegments = m.cleanup(evictionCandidates, touchedSeqNo + 1);
verifyMetadataConsistency(m);
for (SegmentMetadata sm : evictedSegments) {
Assert.assertFalse("Evicted segment was not marked as inactive.", sm.isActive());
}
Assert.assertEquals("Unexpected number of segments were evicted (second-cleanup).", touchedSegments.size(), evictedSegments.size());
for (long segmentId : segments) {
Assert.assertNull("Candidate segment was not evicted (lookup by id)", m.getStreamSegmentMetadata(segmentId));
}
}
Aggregations