Examples with SegmentMetadata io.pravega.segmentstore.server.SegmentMetadata used on opensource projects

Example 41 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StorageWriterTests method verifyFinalOutput.

// region Helpers
private void verifyFinalOutput(HashMap<Long, ByteArrayOutputStream> segmentContents, Collection<Long> transactionIds, TestContext context) {
    // Verify all Transactions are deleted.
    for (long transactionId : transactionIds) {
        SegmentMetadata metadata = context.metadata.getStreamSegmentMetadata(transactionId);
        Assert.assertTrue("Transaction not marked as deleted in metadata: " + transactionId, metadata.isDeleted());
        Assert.assertFalse("Transaction was not deleted from storage after being merged: " + transactionId, context.storage.exists(metadata.getName(), TIMEOUT).join());
        verifyAttributes(metadata, context);
    }
    for (long segmentId : segmentContents.keySet()) {
        SegmentMetadata metadata = context.metadata.getStreamSegmentMetadata(segmentId);
        Assert.assertNotNull("Setup error: No metadata for segment " + segmentId, metadata);
        Assert.assertFalse("Setup error: Not expecting a Transaction segment in the final list: " + segmentId, context.transactionIds.containsKey(metadata.getId()));
        Assert.assertEquals("Metadata does not indicate that all bytes were copied to Storage for segment " + segmentId, metadata.getLength(), metadata.getStorageLength());
        Assert.assertEquals("Metadata.Sealed disagrees with Metadata.SealedInStorage for segment " + segmentId, metadata.isSealed(), metadata.isSealedInStorage());
        SegmentProperties sp = context.storage.getStreamSegmentInfo(metadata.getName(), TIMEOUT).join();
        Assert.assertEquals("Metadata.StorageLength disagrees with Storage.Length for segment " + segmentId, metadata.getStorageLength(), sp.getLength());
        Assert.assertEquals("Metadata.Sealed/SealedInStorage disagrees with Storage.Sealed for segment " + segmentId, metadata.isSealedInStorage(), sp.isSealed());
        byte[] expected = segmentContents.get(segmentId).toByteArray();
        byte[] actual = new byte[expected.length];
        int actualLength = context.storage.read(InMemoryStorage.newHandle(metadata.getName(), true), 0, actual, 0, actual.length, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of bytes read from Storage for segment " + segmentId, metadata.getStorageLength(), actualLength);
        Assert.assertArrayEquals("Unexpected data written to storage for segment " + segmentId, expected, actual);
        Assert.assertEquals("Unexpected truncation offset for segment " + segmentId, metadata.getStartOffset(), context.storage.getTruncationOffset(metadata.getName()));
        verifyAttributes(metadata, context);
    }
}

Example 42 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StorageWriterTests method createTransactions.

private HashMap<Long, ArrayList<Long>> createTransactions(Collection<Long> segmentIds, TestContext context) {
    // Create the Transactions.
    HashMap<Long, ArrayList<Long>> transactions = new HashMap<>();
    long transactionId = Integer.MAX_VALUE;
    for (long parentId : segmentIds) {
        ArrayList<Long> segmentTransactions = new ArrayList<>();
        transactions.put(parentId, segmentTransactions);
        SegmentMetadata parentMetadata = context.metadata.getStreamSegmentMetadata(parentId);
        for (int i = 0; i < TRANSACTIONS_PER_SEGMENT; i++) {
            String transactionName = NameUtils.getTransactionNameFromId(parentMetadata.getName(), UUID.randomUUID());
            context.transactionIds.put(transactionId, parentId);
            context.metadata.mapStreamSegmentId(transactionName, transactionId);
            initializeSegment(transactionId, context);
            segmentTransactions.add(transactionId);
            // Add the operation to the log.
            StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(context.storage.getStreamSegmentInfo(transactionName, TIMEOUT).join());
            mapOp.setStreamSegmentId(transactionId);
            context.dataSource.add(mapOp);
            transactionId++;
        }
    }
    return transactions;
}

Example 43 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class MetadataStoreTestBase method testGetOrAssignStreamSegmentId.

/**
 * Tests the ability of the MetadataStore to generate/return the Id of an existing StreamSegment, as well as
 * retrieving existing attributes.
 */
@Test
public void testGetOrAssignStreamSegmentId() {
    final long baseSegmentId = 1000;
    final long minSegmentLength = 1;
    final int segmentCount = 50;
    Function<String, Long> getSegmentLength = segmentName -> minSegmentLength + MathHelpers.abs(segmentName.hashCode());
    Function<String, Long> getSegmentStartOffset = segmentName -> getSegmentLength.apply(segmentName) / 2;
    @Cleanup TestContext context = createTestContext();
    HashSet<String> segmentNames = new HashSet<>();
    HashSet<String> sealedSegments = new HashSet<>();
    for (int i = 0; i < segmentCount; i++) {
        String segmentName = getName(i);
        segmentNames.add(segmentName);
        val si = StreamSegmentInformation.builder().name(segmentName).length(getSegmentLength.apply(segmentName)).startOffset(getSegmentStartOffset.apply(segmentName)).sealed(i % 2 == 0).attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT))).build();
        if (si.isSealed()) {
            sealedSegments.add(segmentName);
        }
        context.getMetadataStore().updateSegmentInfo(toMetadata(baseSegmentId + i, si), TIMEOUT).join();
    }
    Predicate<String> isSealed = sealedSegments::contains;
    for (String name : segmentNames) {
        long id = context.getMetadataStore().getOrAssignSegmentId(name, TIMEOUT).join();
        Assert.assertNotEquals("No id was assigned for StreamSegment " + name, ContainerMetadata.NO_STREAM_SEGMENT_ID, id);
        SegmentMetadata sm = context.getMetadata().getStreamSegmentMetadata(id);
        Assert.assertNotNull("No metadata was created for StreamSegment " + name, sm);
        long expectedLength = getSegmentLength.apply(name);
        boolean expectedSeal = isSealed.test(name);
        Assert.assertEquals("Metadata does not have the expected length for StreamSegment " + name, expectedLength, sm.getLength());
        Assert.assertEquals("Metadata does not have the expected value for isSealed for StreamSegment " + name, expectedSeal, sm.isSealed());
        val segmentState = context.getMetadataStore().getSegmentInfo(name, TIMEOUT).join();
        Map<AttributeId, Long> expectedAttributes = segmentState == null ? null : segmentState.getAttributes();
        SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in metadata for StreamSegment " + name, expectedAttributes, sm);
        long expectedStartOffset = segmentState == null ? 0 : segmentState.getStartOffset();
        Assert.assertEquals("Unexpected StartOffset in metadata for " + name, expectedStartOffset, sm.getStartOffset());
    }
}

Example 44 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class MetadataCleanerTests method testCleanup.

/**
 * Tests {@link MetadataCleaner#runOnce()}.
 */
@Test
public void testCleanup() throws Exception {
    @Cleanup val context = new TestContext();
    Assert.assertNotEquals(0, context.metadata.getActiveSegmentCount());
    // Cleanup #1. We expect half of the deleted segments to be evicted (due to how they're set up).
    val expected1 = context.metadata.getEvictionCandidates(0, 1000);
    AssertExtensions.assertGreaterThan("Expected at least one eligible segment.", 0, expected1.size());
    context.cleaner.runOnce().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AssertExtensions.assertContainsSameElements("Unexpected evicted segments on the first round.", expected1, context.cleanedUpMetadata, Comparator.comparingLong(SegmentMetadata::getId));
    // Cleanup #2. We expect all the remaining evictable segments to be evicted.
    context.cleanedUpMetadata.clear();
    val expected2 = context.metadata.getEvictionCandidates(context.metadata.getOperationSequenceNumber(), 1000);
    AssertExtensions.assertGreaterThan("Expected at least one eligible segment.", 0, expected2.size());
    context.cleaner.runOnce().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AssertExtensions.assertContainsSameElements("Unexpected evicted segments on the second round.", expected2, context.cleanedUpMetadata, Comparator.comparingLong(SegmentMetadata::getId));
    // Verify that we have properly evicted the attributes that we should have.
    val nonEvictedSegmentIds = context.metadata.getAllStreamSegmentIds();
    AssertExtensions.assertGreaterThan("", 0, nonEvictedSegmentIds.size());
    for (val segmentId : nonEvictedSegmentIds) {
        val sm = context.metadata.getStreamSegmentMetadata(segmentId);
        if (sm.isDeleted() || sm.isMerged()) {
            continue;
        }
        val attributeCount = sm.getAttributes((k, v) -> !Attributes.isCoreAttribute(k)).size();
        assertEquals("Unexpected number of remaining non-core attributes.", CONFIG.getMaxCachedExtendedAttributeCount(), attributeCount);
    }
}

Example 45 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StreamSegmentContainerMetadataTests method testGetEvictionCandidates.

/**
 * Tests the ability to identify Segment Metadatas that are not in use anymore and are eligible for eviction.
 * 1. Creates a number of segments.
 * 2. Truncates repeatedly and at each step verifies that the correct segments were identified as candidates.
 * 3. "Expires" all segments and verifies they are all identified as candidates.
 */
@Test
public void testGetEvictionCandidates() {
    // Expire each segment at a different stage.
    final long firstStageExpiration = SEGMENT_COUNT;
    final long finalExpiration = firstStageExpiration + SEGMENT_COUNT;
    // Create a number of segments.
    // Each segment has a 'LastKnownSequenceNumber' set in incremental order.
    final ArrayList<Long> segments = new ArrayList<>();
    final StreamSegmentContainerMetadata m = new MetadataBuilder(CONTAINER_ID).buildAs();
    populateSegmentsForEviction(segments, m);
    for (int i = 0; i < segments.size(); i++) {
        UpdateableSegmentMetadata segmentMetadata = m.getStreamSegmentMetadata(segments.get(i));
        if (i % 2 == 0) {
            // 1/2 of segments expire at the end.
            segmentMetadata.setLastUsed(finalExpiration);
        } else {
            // The rest of the segments expire in the first stage.
            segmentMetadata.setLastUsed(firstStageExpiration);
        }
    }
    // Add one segment that will be deleted. This should be evicted as soon as its LastUsed is before the truncation point.
    final long deletedSegmentId = segments.size();
    UpdateableSegmentMetadata deletedSegment = m.mapStreamSegmentId(getName(deletedSegmentId), deletedSegmentId);
    deletedSegment.markDeleted();
    deletedSegment.setLastUsed(firstStageExpiration);
    segments.add(deletedSegmentId);
    // Verify that not-yet-truncated operations will not be selected for truncation.
    val truncationPoints = Arrays.asList(0L, firstStageExpiration, finalExpiration, finalExpiration + 1);
    Collection<SegmentMetadata> evictionCandidates;
    for (long truncatedSeqNo : truncationPoints) {
        // Simulate a truncation.
        m.removeTruncationMarkers(truncatedSeqNo);
        // Try to evict everything.
        evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
        checkEvictedSegmentCandidates(evictionCandidates, m, finalExpiration + 1, truncatedSeqNo);
    }
    // Now we expire all segments.
    evictionCandidates = m.getEvictionCandidates(finalExpiration + 1, Integer.MAX_VALUE);
    checkEvictedSegmentCandidates(evictionCandidates, m, finalExpiration + 1, Long.MAX_VALUE);
    // Check that, in the end, all segments in the metadata have been selected for eviction.
    Assert.assertEquals("Not all segments were evicted.", segments.size(), evictionCandidates.size());
}