Examples with SegmentProperties io.pravega.segmentstore.contracts.SegmentProperties used on opensource projects

Example 21 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentContainerTests method testStartOffline.

/**
 * Tests the ability of the StreamSegmentContainer to start in Offline mode (due to an offline DurableLog) and eventually
 * become online when the DurableLog becomes too.
 */
@Test
public void testStartOffline() throws Exception {
    @Cleanup val context = new TestContext();
    AtomicReference<DurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup val dataLogFactory = new TestDurableDataLogFactory(context.dataLogFactory, dataLog::set);
    AtomicReference<OperationLog> durableLog = new AtomicReference<>();
    val durableLogFactory = new WatchableOperationLogFactory(new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, dataLogFactory, executorService()), durableLog::set);
    val containerFactory = new StreamSegmentContainerFactory(DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    // Write some data
    ArrayList<String> segmentNames = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < SEGMENT_COUNT; i++) {
            String segmentName = getSegmentName(i);
            segmentNames.add(segmentName);
            opFutures.add(container.createStreamSegment(segmentName, null, TIMEOUT));
        }
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Disable the DurableDataLog.
        dataLog.get().disable();
    }
    // Start in "Offline" mode, verify operations cannot execute and then shut down - make sure we can shut down an offline container.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        AssertExtensions.assertThrows("append() worked in offline mode.", () -> container.append("foo", new byte[1], null, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("getStreamSegmentInfo() worked in offline mode.", () -> container.getStreamSegmentInfo("foo", false, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("read() worked in offline mode.", () -> container.read("foo", 0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        container.stopAsync().awaitTerminated();
    }
    // Start in "Offline" mode and verify we can resume a normal startup.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        dataLog.get().enable();
        // Wait for the DurableLog to become online.
        durableLog.get().awaitOnline().get(DEFAULT_DURABLE_LOG_CONFIG.getStartRetryDelay().toMillis() * 100, TimeUnit.MILLISECONDS);
        // Verify we can execute regular operations now.
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Verify all operations arrived in Storage.
        ArrayList<CompletableFuture<Void>> segmentsCompletion = new ArrayList<>();
        for (String segmentName : segmentNames) {
            SegmentProperties sp = container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
            segmentsCompletion.add(waitForSegmentInStorage(sp, context));
        }
        Futures.allOf(segmentsCompletion).join();
        container.stopAsync().awaitTerminated();
    }
}

Example 22 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentContainerTests method testMetadataCleanupRecovery.

/**
 * Tests the ability for the SegmentContainer to recover in the following scenario:
 * 1. A segment is created and recorded in the metadata (with some optional operations on it)
 * 2. The segment is evicted from the metadata.
 * 3. The segment is reactivated (with a new metadata mapping). No truncations happened since #2 above.
 * 4. Container shuts down and needs to recover. We need to ensure that recovery succeeds even with the new mapping
 * of the segment.
 */
@Test
public void testMetadataCleanupRecovery() throws Exception {
    final String segmentName = "segment";
    final byte[] appendData = "hello".getBytes();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(250));
    @Cleanup TestContext context = new TestContext(containerConfig);
    val localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    SegmentProperties originalInfo;
    try (val container1 = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService())) {
        container1.startAsync().awaitRunning();
        // Create segment and make one append to it.
        container1.createStreamSegment(segmentName, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        container1.append(segmentName, appendData, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        // Wait until the segment is forgotten.
        container1.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        // Add an append - this will force the segment to be reactivated, thus be registered with a different id.
        container1.append(segmentName, appendData, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        originalInfo = container1.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        container1.stopAsync().awaitTerminated();
    }
    // Restart container and verify it started successfully.
    @Cleanup val container2 = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    container2.startAsync().awaitRunning();
    val recoveredInfo = container2.getStreamSegmentInfo(segmentName, false, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length from recovered segment.", originalInfo.getLength(), recoveredInfo.getLength());
    container2.stopAsync().awaitTerminated();
}

Example 23 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentSeal.

/**
 * Test the seal operation on StreamSegments. Also tests the behavior of Reads (non-tailing) when encountering
 * the end of a sealed StreamSegment.
 */
@Test
public void testSegmentSeal() throws Exception {
    final int appendsPerSegment = 1;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> appendFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    for (String segmentName : segmentNames) {
        ByteArrayOutputStream segmentStream = new ByteArrayOutputStream();
        segmentContents.put(segmentName, segmentStream);
        for (int i = 0; i < appendsPerSegment; i++) {
            byte[] appendData = getAppendData(segmentName, i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            segmentStream.write(appendData);
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Seal first half of segments.
    ArrayList<CompletableFuture<Long>> sealFutures = new ArrayList<>();
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        sealFutures.add(context.container.sealStreamSegment(segmentNames.get(i), TIMEOUT));
    }
    Futures.allOf(sealFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Check that the segments were properly sealed.
    for (int i = 0; i < segmentNames.size(); i++) {
        String segmentName = segmentNames.get(i);
        boolean expectedSealed = i < segmentNames.size() / 2;
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        if (expectedSealed) {
            Assert.assertTrue("Segment is not sealed when it should be " + segmentName, sp.isSealed());
            Assert.assertEquals("Unexpected result from seal() future for segment " + segmentName, sp.getLength(), (long) sealFutures.get(i).join());
            AssertExtensions.assertThrows("Container allowed appending to a sealed segment " + segmentName, context.container.append(segmentName, "foo".getBytes(), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentSealedException);
        } else {
            Assert.assertFalse("Segment is sealed when it shouldn't be " + segmentName, sp.isSealed());
            // Verify we can still append to these segments.
            byte[] appendData = "foo".getBytes();
            context.container.append(segmentName, appendData, null, TIMEOUT).join();
            segmentContents.get(segmentName).write(appendData);
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
        }
    }
    // 4. Reads (regular reads, not tail reads, and only for the sealed segments).
    waitForOperationsInReadIndex(context.container);
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        String segmentName = segmentNames.get(i);
        long segmentLength = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        // Read starting 1 byte from the end - make sure it wont hang at the end by turning into a future read.
        final int totalReadLength = 1;
        long expectedCurrentOffset = segmentLength - totalReadLength;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, Integer.MAX_VALUE, TIMEOUT).join();
        int readLength = 0;
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            if (readEntry.getStreamSegmentOffset() >= segmentLength) {
                Assert.assertEquals("Unexpected value for isEndOfStreamSegment when reaching the end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                AssertExtensions.assertThrows("ReadResultEntry.getContent() returned a result when reached the end of sealed segment " + segmentName, readEntry::getContent, ex -> ex instanceof IllegalStateException);
            } else {
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment before reaching end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                expectedCurrentOffset += readEntryContents.getLength();
                readLength += readEntryContents.getLength();
            }
        }
        Assert.assertEquals("Unexpected number of bytes read.", totalReadLength, readLength);
        Assert.assertTrue("ReadResult was not closed when reaching the end of sealed segment" + segmentName, readResult.isClosed());
    }
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 24 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentContainerTests method testRegularOperationsWithTruncate.

/**
 * Same as testRegularOperations, but truncates the segments as progress is made.
 */
@Test
public void testRegularOperationsWithTruncate() throws Exception {
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    // 2. Add some appends & truncate as we go.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, Long> truncationOffsets = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        for (String segmentName : segmentNames) {
            byte[] appendData = getAppendData(segmentName, i);
            opFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            recordAppend(segmentName, appendData, segmentContents);
            long truncateOffset = truncationOffsets.getOrDefault(segmentName, 0L) + appendData.length / 2 + 1;
            truncationOffsets.put(segmentName, truncateOffset);
            opFutures.add(context.container.truncateStreamSegment(segmentName, truncateOffset, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo
    for (String segmentName : segmentNames) {
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        long expectedStartOffset = truncationOffsets.getOrDefault(segmentName, 0L);
        long expectedLength = lengths.get(segmentName);
        Assert.assertEquals("Unexpected StartOffset for segment " + segmentName, expectedStartOffset, sp.getStartOffset());
        Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength());
        Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
        Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
    }
    // 4. Reads (regular reads, not tail reads).
    checkReadIndex(segmentContents, lengths, truncationOffsets, context);
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 25 with SegmentProperties

use of io.pravega.segmentstore.contracts.SegmentProperties in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentRegularOperations.

/**
 * Tests the createSegment, append, updateAttributes, read, getSegmentInfo, getActiveSegments.
 */
@Test
public void testSegmentRegularOperations() throws Exception {
    final UUID attributeAccumulate = UUID.randomUUID();
    final UUID attributeReplace = UUID.randomUUID();
    final UUID attributeReplaceIfGreater = UUID.randomUUID();
    final UUID attributeReplaceIfEquals = UUID.randomUUID();
    final UUID attributeNoUpdate = UUID.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    checkActiveSegments(context.container, 0);
    activateAllSegments(segmentNames, context);
    checkActiveSegments(context.container, segmentNames.size());
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        for (String segmentName : segmentNames) {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, i == 0 ? AttributeUpdateType.Replace : AttributeUpdateType.ReplaceIfEquals, i + 1, i));
            byte[] appendData = getAppendData(segmentName, i);
            opFutures.add(context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            recordAppend(segmentName, appendData, segmentContents);
        }
    }
    // 2.1 Update some of the attributes.
    for (String segmentName : segmentNames) {
        // Record a one-off update.
        opFutures.add(context.container.updateAttributes(segmentName, Collections.singleton(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, expectedAttributeValue)), TIMEOUT));
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, APPENDS_PER_SEGMENT + i + 1, APPENDS_PER_SEGMENT + i));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo
    for (String segmentName : segmentNames) {
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        long expectedLength = lengths.get(segmentName);
        Assert.assertEquals("Unexpected StartOffset for non-truncated segment " + segmentName, 0, sp.getStartOffset());
        Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength());
        Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
        Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
        // Verify all attribute values.
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeNoUpdate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplace + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplace, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfGreater + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfGreater, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfEquals + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfEquals, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
    }
    checkActiveSegments(context.container, segmentNames.size());
    // 4. Reads (regular reads, not tail reads).
    checkReadIndex(segmentContents, lengths, context);
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}