Examples with MetadataBuilder io.pravega.segmentstore.server.MetadataBuilder used on opensource projects

Example 6 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testDeleteStreamSegment.

/**
 * Tests the ability to delete a StreamSegment from the metadata, as well as any dependent (Transaction) StreamSegments.
 */
@Test
@SuppressWarnings("checkstyle:CyclomaticComplexity")
public void testDeleteStreamSegment() {
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
    final int alreadyDeletedTransactionFrequency = 11;
    ArrayList<Long> segmentIds = new ArrayList<>();
    HashSet<Long> deletedStreamSegmentIds = new HashSet<>();
    for (long i = 0; i < SEGMENT_COUNT; i++) {
        final long segmentId = segmentIds.size();
        segmentIds.add(segmentId);
        m.mapStreamSegmentId(getName(segmentId), segmentId);
        for (long j = 0; j < TRANSACTIONS_PER_SEGMENT_COUNT; j++) {
            final long transactionId = segmentIds.size();
            segmentIds.add(transactionId);
            val tm = m.mapStreamSegmentId(getName(transactionId), transactionId, segmentId);
            if (segmentIds.size() % alreadyDeletedTransactionFrequency == 0) {
                // Mark this transaction as already deleted in Storage.
                tm.markDeleted();
                deletedStreamSegmentIds.add(transactionId);
            } else if (segmentIds.size() % alreadyDeletedTransactionFrequency == 1) {
                // Decoy: this is merged, but not in Storage.
                tm.markMerged();
            }
        }
    }
    // By construction (see above, any index i=3n is a parent StreamSegment, and any index i=3n+1 or 3n+2 is a Transaction).
    // Let's delete a few parent StreamSegments and verify their Transactions are also deleted.
    // Then delete only Transactions, and verify those are the only ones to be deleted.
    final int groupSize = TRANSACTIONS_PER_SEGMENT_COUNT + 1;
    ArrayList<Integer> streamSegmentsToDelete = new ArrayList<>();
    ArrayList<Integer> transactionsToDelete = new ArrayList<>();
    for (int i = 0; i < segmentIds.size(); i++) {
        if (i < segmentIds.size() / 2) {
            // In the first half, we only delete the parents (which will force the Transactions to be deleted too).
            if (i % groupSize == 0) {
                streamSegmentsToDelete.add(i);
            }
        } else {
            // In the second half, we only delete the first Transaction of any segment.
            if (i % groupSize == 1) {
                transactionsToDelete.add(i);
            }
        }
    }
    // Delete stand-alone StreamSegments (and verify Transactions are also deleted).
    for (int index : streamSegmentsToDelete) {
        long segmentId = segmentIds.get(index);
        String name = m.getStreamSegmentMetadata(segmentId).getName();
        Collection<String> expectedDeletedSegmentNames = new ArrayList<>();
        expectedDeletedSegmentNames.add(name);
        deletedStreamSegmentIds.add(segmentId);
        for (int transIndex = 0; transIndex < TRANSACTIONS_PER_SEGMENT_COUNT; transIndex++) {
            long transactionId = segmentIds.get(index + transIndex + 1);
            if (deletedStreamSegmentIds.add(transactionId)) {
                // We only expect a Transaction to be deleted if it hasn't already been deleted.
                expectedDeletedSegmentNames.add(m.getStreamSegmentMetadata(transactionId).getName());
            }
        }
        Collection<String> deletedSegmentNames = extract(m.deleteStreamSegment(name), SegmentMetadata::getName);
        AssertExtensions.assertContainsSameElements("Unexpected StreamSegments were deleted.", expectedDeletedSegmentNames, deletedSegmentNames);
    }
    // Delete Transactions.
    for (int index : transactionsToDelete) {
        long transactionId = segmentIds.get(index);
        String name = m.getStreamSegmentMetadata(transactionId).getName();
        Collection<String> expectedDeletedSegmentNames = new ArrayList<>();
        deletedStreamSegmentIds.add(transactionId);
        expectedDeletedSegmentNames.add(name);
        Collection<String> deletedSegmentNames = extract(m.deleteStreamSegment(name), SegmentMetadata::getName);
        AssertExtensions.assertContainsSameElements("Unexpected StreamSegments were deleted.", expectedDeletedSegmentNames, deletedSegmentNames);
    }
    // Verify deleted segments have not been actually removed from the metadata.
    Collection<Long> metadataSegmentIds = m.getAllStreamSegmentIds();
    AssertExtensions.assertContainsSameElements("Metadata does not contain the expected Segment Ids", segmentIds, metadataSegmentIds);
    // Verify individual StreamSegmentMetadata.
    for (long segmentId : segmentIds) {
        boolean expectDeleted = deletedStreamSegmentIds.contains(segmentId);
        Assert.assertEquals("Unexpected value for isDeleted.", expectDeleted, m.getStreamSegmentMetadata(segmentId).isDeleted());
    }
}

Example 7 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testMapStreamSegment.

/**
 * Tests the ability to map new StreamSegments (as well as Transactions).
 */
@Test
public void testMapStreamSegment() {
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
    final HashMap<Long, Long> segmentIds = new HashMap<>();
    for (long i = 0; i < SEGMENT_COUNT; i++) {
        final long segmentId = segmentIds.size();
        String segmentName = getName(segmentId);
        // This should work.
        // Change the sequence number, before mapping.
        m.nextOperationSequenceNumber();
        m.mapStreamSegmentId(segmentName, segmentId);
        segmentIds.put(segmentId, m.getOperationSequenceNumber());
        Assert.assertEquals("Unexpected value from getStreamSegmentId (Stand-alone Segment).", segmentId, m.getStreamSegmentId(segmentName, false));
        // Now check that we cannot re-map the same SegmentId or SegmentName.
        AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping the same SegmentId twice.", () -> m.mapStreamSegmentId(segmentName + "foo", segmentId), ex -> ex instanceof IllegalArgumentException);
        AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping the same SegmentName twice.", () -> m.mapStreamSegmentId(segmentName, segmentId + 1), ex -> ex instanceof IllegalArgumentException);
        for (long j = 0; j < TRANSACTIONS_PER_SEGMENT_COUNT; j++) {
            final long transactionId = segmentIds.size();
            String transactionName = getName(transactionId);
            AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping a Transaction to an inexistent parent.", () -> m.mapStreamSegmentId(transactionName, transactionId, transactionId), ex -> ex instanceof IllegalArgumentException);
            // This should work.
            // Change the sequence number, before mapping.
            m.nextOperationSequenceNumber();
            m.mapStreamSegmentId(transactionName, transactionId, segmentId);
            segmentIds.put(transactionId, m.getOperationSequenceNumber());
            Assert.assertEquals("Unexpected value from getStreamSegmentId (Transaction Segment).", transactionId, m.getStreamSegmentId(transactionName, false));
            // Now check that we cannot re-map the same Transaction Id or Name.
            AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping the same Transaction SegmentId twice.", () -> m.mapStreamSegmentId(transactionName + "foo", transactionId, segmentId), ex -> ex instanceof IllegalArgumentException);
            AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping the same Transaction SegmentName twice.", () -> m.mapStreamSegmentId(transactionName, transactionId + 1, segmentId), ex -> ex instanceof IllegalArgumentException);
            // Now check that we cannot map a Transaction to another Transaction.
            AssertExtensions.assertThrows("mapStreamSegmentId allowed mapping the a Transaction to another Transaction.", () -> m.mapStreamSegmentId(transactionName + "foo", transactionId + 1, transactionId), ex -> ex instanceof IllegalArgumentException);
        }
    }
    // Check getLastUsed.
    for (Map.Entry<Long, Long> e : segmentIds.entrySet()) {
        // Increment the SeqNo so we can verify 'updateLastUsed'.
        m.nextOperationSequenceNumber();
        SegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(e.getKey());
        Assert.assertEquals("Unexpected value for getLastUsed for untouched segment.", (long) e.getValue(), segmentMetadata.getLastUsed());
        m.getStreamSegmentId(segmentMetadata.getName(), false);
        Assert.assertEquals("Unexpected value for getLastUsed for untouched segment.", (long) e.getValue(), segmentMetadata.getLastUsed());
        m.getStreamSegmentId(segmentMetadata.getName(), true);
        Assert.assertEquals("Unexpected value for getLastUsed for touched segment.", m.getOperationSequenceNumber(), segmentMetadata.getLastUsed());
    }
    Collection<Long> metadataSegmentIds = m.getAllStreamSegmentIds();
    AssertExtensions.assertContainsSameElements("Metadata does not contain the expected Segment Ids", segmentIds.keySet(), metadataSegmentIds);
}

Example 8 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testMaxActiveSegmentCount.

/**
 * Tests the ability of the metadata to enforce the Maximum Active Segment Count rule.
 */
@Test
public void testMaxActiveSegmentCount() {
    final int maxCount = 2;
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).withMaxActiveSegmentCount(maxCount).build();
    // Map 1 segment + 1 transactions. These should fill up the capacity.
    m.mapStreamSegmentId("1", 1);
    m.mapStreamSegmentId("2", 2, 1);
    // Verify we cannot map anything now.
    AssertExtensions.assertThrows("Metadata allowed mapping more segments than indicated (segment).", () -> m.mapStreamSegmentId("3", 3), ex -> ex instanceof IllegalStateException);
    AssertExtensions.assertThrows("Metadata allowed mapping more segments than indicated (transaction).", () -> m.mapStreamSegmentId("3", 3, 1), ex -> ex instanceof IllegalStateException);
    // Verify we are allowed to do this in recovery mode.
    m.enterRecoveryMode();
    m.mapStreamSegmentId("3", 3);
    m.mapStreamSegmentId("4", 4, 3);
    m.exitRecoveryMode();
    Assert.assertNotNull("Metadata did not map new segment that exceeded the quota in recovery mode.", m.getStreamSegmentMetadata(3));
    Assert.assertNotNull("Metadata did not map new transaction that exceeded the quota in recovery mode.", m.getStreamSegmentMetadata(4));
}

Example 9 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testReset.

/**
 * Tests the ability for the metadata to reset itself.
 */
@Test
public void testReset() {
    // Segments, Sequence Number + Truncation markers
    final UpdateableContainerMetadata m = new MetadataBuilder(CONTAINER_ID).build();
    // Set a high Sequence Number
    m.enterRecoveryMode();
    m.setOperationSequenceNumber(Integer.MAX_VALUE);
    m.setContainerEpoch(Integer.MAX_VALUE + 1L);
    m.exitRecoveryMode();
    // Populate some StreamSegments.
    ArrayList<Long> segmentIds = new ArrayList<>();
    for (long i = 0; i < SEGMENT_COUNT; i++) {
        final long segmentId = segmentIds.size();
        segmentIds.add(segmentId);
        m.mapStreamSegmentId(getName(segmentId), segmentId);
        for (long j = 0; j < TRANSACTIONS_PER_SEGMENT_COUNT; j++) {
            final long transactionId = segmentIds.size();
            segmentIds.add(transactionId);
            m.mapStreamSegmentId(getName(transactionId), transactionId, segmentId);
        }
    }
    // Add some truncation markers.
    final long truncationMarkerSeqNo = 10;
    m.recordTruncationMarker(truncationMarkerSeqNo, new TestLogAddress(truncationMarkerSeqNo));
    m.setValidTruncationPoint(truncationMarkerSeqNo);
    AssertExtensions.assertThrows("reset() worked in non-recovery mode.", m::reset, ex -> ex instanceof IllegalStateException);
    // Do the reset.
    m.enterRecoveryMode();
    m.reset();
    m.exitRecoveryMode();
    // Verify everything was reset.
    Assert.assertEquals("Sequence Number was not reset.", ContainerMetadata.INITIAL_OPERATION_SEQUENCE_NUMBER, m.getOperationSequenceNumber());
    AssertExtensions.assertLessThan("Epoch was not reset.", 0, m.getContainerEpoch());
    for (long segmentId : segmentIds) {
        Assert.assertEquals("SegmentMetadata was not reset (getStreamSegmentId).", ContainerMetadata.NO_STREAM_SEGMENT_ID, m.getStreamSegmentId(getName(segmentId), false));
        Assert.assertNull("SegmentMetadata was not reset (getStreamSegmentMetadata).", m.getStreamSegmentMetadata(segmentId));
    }
    LogAddress tmSeqNo = m.getClosestTruncationMarker(truncationMarkerSeqNo);
    Assert.assertNull("Truncation Markers were not reset.", tmSeqNo);
    Assert.assertFalse("Truncation Points were not reset.", m.isValidTruncationPoint(truncationMarkerSeqNo));
}

Example 10 with MetadataBuilder

use of io.pravega.segmentstore.server.MetadataBuilder in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testCleanupCapped.

/**
 * Tests the ability to evict Segment Metadata instances that are not in use anymore, subject to a cap. This test
 * focuses in particular to the case when a segment and all its transactions are eligible for removal, however due
 * to the cap, some transactions are no longer eligible, and thus the parent segment must not be evicted either.
 */
@Test
public void testCleanupCapped() {
    final int maxCap = 5;
    final int txnCount = maxCap * 2 + 1;
    // Expire each Segment at a different stage.
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    // Create a single parent segment, followed by a number of transactions. Each segment has a 'LastUsed' set in
    // incremental order, with the Parent Segment being the least recently used.
    long maxLastUsed = 1;
    val segments = new ArrayList<Long>();
    final long parentSegmentId = segments.size();
    segments.add(parentSegmentId);
    m.mapStreamSegmentId(getName(parentSegmentId), parentSegmentId).setLastUsed(maxLastUsed++);
    for (int i = 0; i < txnCount; i++) {
        final long transactionId = segments.size();
        segments.add(transactionId);
        m.mapStreamSegmentId("Transaction_" + transactionId, transactionId, parentSegmentId).setLastUsed(maxLastUsed++);
    }
    // Collect a number of eviction candidates using a stringent max cap (less than the number of segments),
    // then evict those segments.
    m.removeTruncationMarkers(maxLastUsed);
    val evictionCandidates = m.getEvictionCandidates(maxLastUsed, maxCap);
    val evictedSegments = m.cleanup(evictionCandidates, maxLastUsed);
    AssertExtensions.assertGreaterThan("At least one segment was expected to be evicted.", 0, evictedSegments.size());
    // Validate ContainerMetadata integrity.
    verifyMetadataConsistency(m);
    Assert.assertNotNull("Not expecting the parent Segment to be evicted.", m.getStreamSegmentMetadata(parentSegmentId));
}