Examples with SegmentProperties io.pravega.segmentstore.contracts.SegmentProperties used on opensource projects

Example 26 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentMapperTests method assertSegmentCreated.

private void assertSegmentCreated(String segmentName, Collection<AttributeUpdate> attributeUpdates, TestContext context) {
    SegmentProperties sp = context.storage.getStreamSegmentInfo(segmentName, TIMEOUT).join();
    Assert.assertNotNull("No segment has been created in the Storage for " + segmentName, sp);
    long segmentId = context.metadata.getStreamSegmentId(segmentName, false);
    Assert.assertEquals("Segment '" + segmentName + "' has been registered in the metadata.", ContainerMetadata.NO_STREAM_SEGMENT_ID, segmentId);
    val attributes = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    val actualAttributes = context.stateStore.get(segmentName, TIMEOUT).join().getAttributes();
    AssertExtensions.assertMapEquals("Wrong attributes.", attributes, actualAttributes);
}

Example 27 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class DurableLogTests method testRecoveryWithMetadataCleanup.

/**
 * Tests the following recovery scenario:
 * 1. A Segment is created and recorded in the metadata with some optional operations executing on it.
 * 2. The segment is evicted from the metadata.
 * 3. The segment is reactivated (with a new metadata mapping) - possibly due to an append. No truncation since #2.
 * 4. Recovery.
 */
@Test
public void testRecoveryWithMetadataCleanup() throws Exception {
    final long truncatedSeqNo = Integer.MAX_VALUE;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    long segmentId;
    // First DurableLog. We use this for generating data.
    val metadata1 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    SegmentProperties originalSegmentInfo;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create the segment.
        val segmentIds = createStreamSegmentsWithOperations(1, metadata1, durableLog, storage);
        segmentId = segmentIds.stream().findFirst().orElse(-1L);
        // Evict the segment.
        val sm1 = metadata1.getStreamSegmentMetadata(segmentId);
        originalSegmentInfo = sm1.getSnapshot();
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata1.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata1.cleanup(Collections.singleton(sm1), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // Map the segment again.
        val reMapOp = new StreamSegmentMapOperation(originalSegmentInfo);
        reMapOp.setStreamSegmentId(segmentId);
        durableLog.add(reMapOp, TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #1. This should work well.
    val metadata2 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Get segment info
        val recoveredSegmentInfo = metadata1.getStreamSegmentMetadata(segmentId).getSnapshot();
        Assert.assertEquals("Unexpected length from recovered segment.", originalSegmentInfo.getLength(), recoveredSegmentInfo.getLength());
        // Now evict the segment again ...
        val sm = metadata2.getStreamSegmentMetadata(segmentId);
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata2.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata2.cleanup(Collections.singleton(sm), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // ... and re-map it with a new Id. This is a perfectly valid operation, and we can't prevent it.
        durableLog.add(new StreamSegmentMapOperation(originalSegmentInfo), TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #2. This should fail due to the same segment mapped multiple times with different ids.
    val metadata3 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata3, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata3, dataLogFactory, readIndex, executorService())) {
        AssertExtensions.assertThrows("Recovery did not fail with the expected exception in case of multi-mapping", () -> durableLog.startAsync().awaitRunning(), ex -> ex instanceof IllegalStateException && ex.getCause() instanceof DataCorruptionException && ex.getCause().getCause() instanceof MetadataUpdateException);
    }
}

Example 28 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentStoreTestBase method waitForSegmentInStorage.

private CompletableFuture<Void> waitForSegmentInStorage(SegmentProperties sp, StreamSegmentStore readOnlyStore) {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    AtomicBoolean tryAgain = new AtomicBoolean(true);
    return Futures.loop(tryAgain::get, () -> readOnlyStore.getStreamSegmentInfo(sp.getName(), false, TIMEOUT).thenCompose(storageProps -> {
        if (sp.isSealed()) {
            tryAgain.set(!storageProps.isSealed());
        } else {
            tryAgain.set(sp.getLength() != storageProps.getLength());
        }
        if (tryAgain.get() && !timer.hasRemaining()) {
            return Futures.<Void>failedFuture(new TimeoutException(String.format("Segment %s did not complete in Storage in the allotted time.", sp.getName())));
        } else {
            return Futures.delayedFuture(Duration.ofMillis(100), executorService());
        }
    }), executorService());
}

Example 29 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentStoreTestBase method waitForSegmentsInStorage.

private CompletableFuture<Void> waitForSegmentsInStorage(Collection<String> segmentNames, StreamSegmentStore baseStore, StreamSegmentStore readOnlyStore) {
    ArrayList<CompletableFuture<Void>> segmentsCompletion = new ArrayList<>();
    for (String segmentName : segmentNames) {
        SegmentProperties sp = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        segmentsCompletion.add(waitForSegmentInStorage(sp, readOnlyStore));
    }
    return Futures.allOf(segmentsCompletion);
}

Example 30 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkStorage.

private static void checkStorage(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore baseStore, StreamSegmentStore readOnlySegmentStore) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        // 1. Deletion status
        SegmentProperties sp = null;
        try {
            sp = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        } catch (Exception ex) {
            if (!(Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException)) {
                throw ex;
            }
        }
        if (sp == null) {
            AssertExtensions.assertThrows("Segment is marked as deleted in SegmentStore but was not deleted in Storage " + segmentName, () -> readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException);
            // No need to do other checks.
            continue;
        }
        // 2. Seal Status
        SegmentProperties storageProps = readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Segment seal status disagree between Store and Storage for segment " + segmentName, sp.isSealed(), storageProps.isSealed());
        // 3. Contents.
        SegmentProperties metadataProps = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Storage length for segment " + segmentName, expectedData.length, storageProps.getLength());
        byte[] actualData = new byte[expectedData.length];
        int actualLength = 0;
        int expectedLength = actualData.length;
        try {
            @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, 0, actualData.length, TIMEOUT).join();
            actualLength = readResult.readRemaining(actualData, TIMEOUT);
        } catch (Exception ex) {
            ex = (Exception) Exceptions.unwrap(ex);
            if (!(ex instanceof StreamSegmentTruncatedException) || metadataProps.getStartOffset() == 0) {
                // We encountered an unexpected Exception, or a Truncated Segment which was not expected to be truncated.
                throw ex;
            }
            // Read from the truncated point, except if the whole segment got truncated.
            expectedLength = (int) (storageProps.getLength() - metadataProps.getStartOffset());
            if (metadataProps.getStartOffset() < storageProps.getLength()) {
                @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join();
                actualLength = readResult.readRemaining(actualData, TIMEOUT);
            }
        }
        Assert.assertEquals("Unexpected number of bytes read from Storage for segment " + segmentName, expectedLength, actualLength);
        AssertExtensions.assertArrayEquals("Unexpected data written to storage for segment " + segmentName, expectedData, expectedData.length - expectedLength, actualData, 0, expectedLength);
    }
}