Examples with StreamSegmentTruncatedException io.pravega.segmentstore.contracts.StreamSegmentTruncatedException used on opensource projects

Example 1 with StreamSegmentTruncatedException

use of io.pravega.segmentstore.contracts.StreamSegmentTruncatedException in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReadsWhileTruncating.

private void checkReadsWhileTruncating(HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> startOffsets, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        long expectedCurrentOffset = 0;
        boolean truncate = false;
        while (expectedCurrentOffset < segmentLength) {
            @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT).join();
            Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
            // We only test the truncation-related pieces here; other read-related checks are done in checkReads.
            while (readResult.hasNext()) {
                ReadResultEntry readEntry = readResult.next();
                Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
                if (!readEntry.getContent().isDone()) {
                    readEntry.requestContent(TIMEOUT);
                }
                if (readEntry.getType() == ReadResultEntryType.Truncated) {
                    long startOffset = startOffsets.getOrDefault(segmentName, 0L);
                    // Verify that the Segment actually is truncated beyond this offset.
                    AssertExtensions.assertLessThan("Found Truncated ReadResultEntry but current offset not truncated.", startOffset, readEntry.getStreamSegmentOffset());
                    // Verify the ReadResultEntry cannot be used and throws an appropriate exception.
                    AssertExtensions.assertThrows("ReadEntry.getContent() did not throw for a Truncated entry.", readEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
                    // Verify ReadResult is done.
                    Assert.assertFalse("Unexpected result from ReadResult.hasNext when encountering truncated entry.", readResult.hasNext());
                    // Verify attempting to read at the current offset will return the appropriate entry (and not throw).
                    @Cleanup ReadResult truncatedResult = store.read(segmentName, readEntry.getStreamSegmentOffset(), 1, TIMEOUT).join();
                    val first = truncatedResult.next();
                    Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType());
                    // Skip over until the first non-truncated offset.
                    expectedCurrentOffset = Math.max(expectedCurrentOffset, startOffset);
                    continue;
                }
                // Non-truncated entry; do the usual verifications.
                readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                byte[] actualData = new byte[readEntryContents.getLength()];
                StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
                AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
                expectedCurrentOffset += readEntryContents.getLength();
                // Every other read, determine if we should truncate or not.
                if (truncate) {
                    long truncateOffset;
                    if (segmentName.hashCode() % 2 == 0) {
                        // Truncate just beyond the current read offset.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset + 1);
                    } else {
                        // Truncate half of what we read so far.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset / 2 + 1);
                    }
                    startOffsets.put(segmentName, truncateOffset);
                    store.truncateStreamSegment(segmentName, truncateOffset, TIMEOUT).join();
                }
                truncate = !truncate;
            }
            Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
        }
    }
}

Example 2 with StreamSegmentTruncatedException

use of io.pravega.segmentstore.contracts.StreamSegmentTruncatedException in project pravega by pravega.

the class ContainerReadIndexTests method checkReadIndex.

private void checkReadIndex(String testId, HashMap<Long, ByteArrayOutputStream> segmentContents, TestContext context) throws Exception {
    for (long segmentId : segmentContents.keySet()) {
        long startOffset = context.metadata.getStreamSegmentMetadata(segmentId).getStartOffset();
        long segmentLength = context.metadata.getStreamSegmentMetadata(segmentId).getLength();
        byte[] expectedData = segmentContents.get(segmentId).toByteArray();
        if (startOffset > 0) {
            @Cleanup ReadResult truncatedResult = context.readIndex.read(segmentId, 0, 1, TIMEOUT);
            val first = truncatedResult.next();
            Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType());
            AssertExtensions.assertThrows("Truncate ReadResultEntryType did not throw when getContent() was invoked.", () -> {
                first.requestContent(TIMEOUT);
                return first.getContent();
            }, ex -> ex instanceof StreamSegmentTruncatedException);
        }
        long expectedCurrentOffset = startOffset;
        @Cleanup ReadResult readResult = context.readIndex.read(segmentId, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT);
        Assert.assertTrue(testId + ": Empty read result for segment " + segmentId, readResult.hasNext());
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan(testId + ": getRequestedReadLength should be a positive integer for segment " + segmentId, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals(testId + ": Unexpected value from getStreamSegmentOffset for segment " + segmentId, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            // Since this is a non-sealed segment, we only expect Cache or Storage read result entries.
            Assert.assertTrue(testId + ": Unexpected type of ReadResultEntry for non-sealed segment " + segmentId, readEntry.getType() == ReadResultEntryType.Cache || readEntry.getType() == ReadResultEntryType.Storage);
            if (readEntry.getType() == ReadResultEntryType.Cache) {
                Assert.assertTrue(testId + ": getContent() did not return a completed future (ReadResultEntryType.Cache) for segment" + segmentId, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            } else if (readEntry.getType() == ReadResultEntryType.Storage) {
                Assert.assertFalse(testId + ": getContent() did not return a non-completed future (ReadResultEntryType.Storage) for segment" + segmentId, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            }
            // Request content, in case it wasn't returned yet.
            readEntry.requestContent(TIMEOUT);
            ReadResultEntryContents readEntryContents = readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            AssertExtensions.assertGreaterThan(testId + ": getContent() returned an empty result entry for segment " + segmentId, 0, readEntryContents.getLength());
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals(testId + ": Unexpected data read from segment " + segmentId + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue(testId + ": ReadResult was not closed post-full-consumption for segment" + segmentId, readResult.isClosed());
    }
}

Example 3 with StreamSegmentTruncatedException

use of io.pravega.segmentstore.contracts.StreamSegmentTruncatedException in project pravega by pravega.

the class RollingStorage method checkTruncatedSegment.

@SneakyThrows(StreamingException.class)
private void checkTruncatedSegment(StreamingException ex, RollingSegmentHandle handle, SegmentChunk segmentChunk) {
    if (ex != null && (Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) || !segmentChunk.exists()) {
        // We ran into a SegmentChunk that does not exist (either marked as such or due to a failed read).
        segmentChunk.markInexistent();
        String message = String.format("Offsets %d-%d have been deleted.", segmentChunk.getStartOffset(), segmentChunk.getLastOffset());
        ex = new StreamSegmentTruncatedException(handle.getSegmentName(), message, ex);
    }
    if (ex != null) {
        throw ex;
    }
}

Example 4 with StreamSegmentTruncatedException

use of io.pravega.segmentstore.contracts.StreamSegmentTruncatedException in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkStorage.

private static void checkStorage(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore baseStore, StreamSegmentStore readOnlySegmentStore) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        // 1. Deletion status
        SegmentProperties sp = null;
        try {
            sp = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        } catch (Exception ex) {
            if (!(Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException)) {
                throw ex;
            }
        }
        if (sp == null) {
            AssertExtensions.assertThrows("Segment is marked as deleted in SegmentStore but was not deleted in Storage " + segmentName, () -> readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException);
            // No need to do other checks.
            continue;
        }
        // 2. Seal Status
        SegmentProperties storageProps = readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Segment seal status disagree between Store and Storage for segment " + segmentName, sp.isSealed(), storageProps.isSealed());
        // 3. Contents.
        SegmentProperties metadataProps = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Storage length for segment " + segmentName, expectedData.length, storageProps.getLength());
        byte[] actualData = new byte[expectedData.length];
        int actualLength = 0;
        int expectedLength = actualData.length;
        try {
            @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, 0, actualData.length, TIMEOUT).join();
            actualLength = readResult.readRemaining(actualData, TIMEOUT);
        } catch (Exception ex) {
            ex = (Exception) Exceptions.unwrap(ex);
            if (!(ex instanceof StreamSegmentTruncatedException) || metadataProps.getStartOffset() == 0) {
                // We encountered an unexpected Exception, or a Truncated Segment which was not expected to be truncated.
                throw ex;
            }
            // Read from the truncated point, except if the whole segment got truncated.
            expectedLength = (int) (storageProps.getLength() - metadataProps.getStartOffset());
            if (metadataProps.getStartOffset() < storageProps.getLength()) {
                @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join();
                actualLength = readResult.readRemaining(actualData, TIMEOUT);
            }
        }
        Assert.assertEquals("Unexpected number of bytes read from Storage for segment " + segmentName, expectedLength, actualLength);
        AssertExtensions.assertArrayEquals("Unexpected data written to storage for segment " + segmentName, expectedData, expectedData.length - expectedLength, actualData, 0, expectedLength);
    }
}

Example 5 with StreamSegmentTruncatedException

use of io.pravega.segmentstore.contracts.StreamSegmentTruncatedException in project pravega by pravega.

the class ContainerReadIndexTests method testTruncateConcurrently.

/**
 * Tests a scenario of truncation that happens concurrently with reading (segment is truncated while reading).
 */
@Test
public void testTruncateConcurrently() throws Exception {
    @Cleanup TestContext context = new TestContext();
    List<Long> segmentIds = createSegments(context).subList(0, 1);
    long segmentId = segmentIds.get(0);
    ByteArrayOutputStream segmentContents = new ByteArrayOutputStream();
    appendData(segmentIds, Collections.singletonMap(segmentId, segmentContents), context);
    // Begin a read result.
    UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
    @Cleanup ReadResult rr = context.readIndex.read(segmentId, 0, (int) sm.getLength(), TIMEOUT);
    ReadResultEntry firstEntry = rr.next();
    firstEntry.requestContent(TIMEOUT);
    int firstEntryLength = firstEntry.getContent().join().getLength();
    AssertExtensions.assertLessThan("Unexpected length of the first read result entry.", sm.getLength(), firstEntryLength);
    // Truncate the segment just after the end of the first returned read result.
    sm.setStartOffset(firstEntryLength + 1);
    ReadResultEntry secondEntry = rr.next();
    Assert.assertTrue("Unexpected ReadResultEntryType.isTerminal of truncated result entry.", secondEntry.getType().isTerminal());
    Assert.assertEquals("Unexpected ReadResultEntryType of truncated result entry.", ReadResultEntryType.Truncated, secondEntry.getType());
    AssertExtensions.assertThrows("Expecting getContent() to return a failed CompletableFuture.", secondEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
    Assert.assertFalse("Unexpected result from hasNext after processing terminal result entry.", rr.hasNext());
}