Examples with ReadResult io.pravega.segmentstore.contracts.ReadResult used on opensource projects

Example 1 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReadsWhileTruncating.

private void checkReadsWhileTruncating(HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> startOffsets, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        long expectedCurrentOffset = 0;
        boolean truncate = false;
        while (expectedCurrentOffset < segmentLength) {
            @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT).join();
            Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
            // We only test the truncation-related pieces here; other read-related checks are done in checkReads.
            while (readResult.hasNext()) {
                ReadResultEntry readEntry = readResult.next();
                Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
                if (!readEntry.getContent().isDone()) {
                    readEntry.requestContent(TIMEOUT);
                }
                if (readEntry.getType() == ReadResultEntryType.Truncated) {
                    long startOffset = startOffsets.getOrDefault(segmentName, 0L);
                    // Verify that the Segment actually is truncated beyond this offset.
                    AssertExtensions.assertLessThan("Found Truncated ReadResultEntry but current offset not truncated.", startOffset, readEntry.getStreamSegmentOffset());
                    // Verify the ReadResultEntry cannot be used and throws an appropriate exception.
                    AssertExtensions.assertThrows("ReadEntry.getContent() did not throw for a Truncated entry.", readEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
                    // Verify ReadResult is done.
                    Assert.assertFalse("Unexpected result from ReadResult.hasNext when encountering truncated entry.", readResult.hasNext());
                    // Verify attempting to read at the current offset will return the appropriate entry (and not throw).
                    @Cleanup ReadResult truncatedResult = store.read(segmentName, readEntry.getStreamSegmentOffset(), 1, TIMEOUT).join();
                    val first = truncatedResult.next();
                    Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType());
                    // Skip over until the first non-truncated offset.
                    expectedCurrentOffset = Math.max(expectedCurrentOffset, startOffset);
                    continue;
                }
                // Non-truncated entry; do the usual verifications.
                readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                byte[] actualData = new byte[readEntryContents.getLength()];
                StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
                AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
                expectedCurrentOffset += readEntryContents.getLength();
                // Every other read, determine if we should truncate or not.
                if (truncate) {
                    long truncateOffset;
                    if (segmentName.hashCode() % 2 == 0) {
                        // Truncate just beyond the current read offset.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset + 1);
                    } else {
                        // Truncate half of what we read so far.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset / 2 + 1);
                    }
                    startOffsets.put(segmentName, truncateOffset);
                    store.truncateStreamSegment(segmentName, truncateOffset, TIMEOUT).join();
                }
                truncate = !truncate;
            }
            Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
        }
    }
}

Example 2 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReads.

private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength);
        long expectedCurrentOffset = 0;
        @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) segmentLength, TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in StreamSegmentContainerTests; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            if (!readEntry.getContent().isDone()) {
                readEntry.requestContent(TIMEOUT);
            }
            readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            ReadResultEntryContents readEntryContents = readEntry.getContent().join();
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
    }
}

Example 3 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentContainerTests method checkReadIndex.

private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, TestContext context) throws Exception {
    waitForOperationsInReadIndex(context.container);
    for (String segmentName : segmentContents.keySet()) {
        long segmentLength = lengths.get(segmentName);
        long startOffset = truncationOffsets.getOrDefault(segmentName, 0L);
        val si = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset());
        Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength());
        byte[] expectedData = segmentContents.get(segmentName).toByteArray();
        long expectedCurrentOffset = startOffset;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            ReadResultEntryContents readEntryContents = readEntry.getContent().join();
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
    }
}

Example 4 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentDelete.

/**
 * Tests the ability to delete StreamSegments.
 */
@Test
public void testSegmentDelete() throws Exception {
    final int appendsPerSegment = 1;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> appendFutures = new ArrayList<>();
    for (int i = 0; i < appendsPerSegment; i++) {
        for (String segmentName : segmentNames) {
            appendFutures.add(context.container.append(segmentName, getAppendData(segmentName, i), null, TIMEOUT));
            for (String transactionName : transactionsBySegment.get(segmentName)) {
                appendFutures.add(context.container.append(transactionName, getAppendData(transactionName, i), null, TIMEOUT));
            }
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Delete the first half of the segments.
    ArrayList<CompletableFuture<Void>> deleteFutures = new ArrayList<>();
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        String segmentName = segmentNames.get(i);
        deleteFutures.add(context.container.deleteStreamSegment(segmentName, TIMEOUT));
    }
    Futures.allOf(deleteFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Verify that only the first half of the segments (and their Transactions) were deleted, and not the others.
    for (int i = 0; i < segmentNames.size(); i++) {
        ArrayList<String> toCheck = new ArrayList<>();
        toCheck.add(segmentNames.get(i));
        toCheck.addAll(transactionsBySegment.get(segmentNames.get(i)));
        boolean expectedDeleted = i < segmentNames.size() / 2;
        if (expectedDeleted) {
            // Verify the segments and their Transactions are not there anymore.
            for (String sn : toCheck) {
                AssertExtensions.assertThrows("getStreamSegmentInfo did not throw expected exception when called on a deleted StreamSegment.", context.container.getStreamSegmentInfo(sn, false, TIMEOUT)::join, ex -> ex instanceof StreamSegmentNotExistsException);
                AssertExtensions.assertThrows("append did not throw expected exception when called on a deleted StreamSegment.", context.container.append(sn, "foo".getBytes(), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentNotExistsException);
                AssertExtensions.assertThrows("read did not throw expected exception when called on a deleted StreamSegment.", context.container.read(sn, 0, 1, TIMEOUT)::join, ex -> ex instanceof StreamSegmentNotExistsException);
                Assert.assertFalse("Segment not deleted in storage.", context.storage.exists(sn, TIMEOUT).join());
            }
        } else {
            // Verify the segments and their Transactions are still there.
            for (String sn : toCheck) {
                SegmentProperties props = context.container.getStreamSegmentInfo(sn, false, TIMEOUT).join();
                Assert.assertFalse("Not-deleted segment (or one of its Transactions) was marked as deleted in metadata.", props.isDeleted());
                // Verify we can still append and read from this segment.
                context.container.append(sn, "foo".getBytes(), null, TIMEOUT).join();
                @Cleanup ReadResult rr = context.container.read(sn, 0, 1, TIMEOUT).join();
                // Verify the segment still exists in storage.
                context.storage.getStreamSegmentInfo(sn, TIMEOUT).join();
            }
        }
    }
    context.container.stopAsync().awaitTerminated();
}

Example 5 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class OperationLogTestBase method performReadIndexChecks.

void performReadIndexChecks(Collection<OperationWithCompletion> operations, ReadIndex readIndex) throws Exception {
    AbstractMap<Long, Integer> expectedLengths = getExpectedLengths(operations);
    AbstractMap<Long, InputStream> expectedData = getExpectedContents(operations);
    for (Map.Entry<Long, InputStream> e : expectedData.entrySet()) {
        int expectedLength = expectedLengths.getOrDefault(e.getKey(), -1);
        @Cleanup ReadResult readResult = readIndex.read(e.getKey(), 0, expectedLength, TIMEOUT);
        int readLength = 0;
        while (readResult.hasNext()) {
            ReadResultEntryContents entry = readResult.next().getContent().join();
            int length = entry.getLength();
            readLength += length;
            int streamSegmentOffset = expectedLengths.getOrDefault(e.getKey(), 0);
            expectedLengths.put(e.getKey(), streamSegmentOffset + length);
            AssertExtensions.assertStreamEquals(String.format("Unexpected data returned from ReadIndex. StreamSegmentId = %d, Offset = %d.", e.getKey(), streamSegmentOffset), e.getValue(), entry.getData(), length);
        }
        Assert.assertEquals("Not enough bytes were read from the ReadIndex for StreamSegment " + e.getKey(), expectedLength, readLength);
    }
}