Examples with ReadResultEntry io.pravega.segmentstore.contracts.ReadResultEntry used on opensource projects

Example 1 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReadsWhileTruncating.

private void checkReadsWhileTruncating(HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> startOffsets, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        long expectedCurrentOffset = 0;
        boolean truncate = false;
        while (expectedCurrentOffset < segmentLength) {
            @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT).join();
            Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
            // We only test the truncation-related pieces here; other read-related checks are done in checkReads.
            while (readResult.hasNext()) {
                ReadResultEntry readEntry = readResult.next();
                Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
                if (!readEntry.getContent().isDone()) {
                    readEntry.requestContent(TIMEOUT);
                }
                if (readEntry.getType() == ReadResultEntryType.Truncated) {
                    long startOffset = startOffsets.getOrDefault(segmentName, 0L);
                    // Verify that the Segment actually is truncated beyond this offset.
                    AssertExtensions.assertLessThan("Found Truncated ReadResultEntry but current offset not truncated.", startOffset, readEntry.getStreamSegmentOffset());
                    // Verify the ReadResultEntry cannot be used and throws an appropriate exception.
                    AssertExtensions.assertThrows("ReadEntry.getContent() did not throw for a Truncated entry.", readEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
                    // Verify ReadResult is done.
                    Assert.assertFalse("Unexpected result from ReadResult.hasNext when encountering truncated entry.", readResult.hasNext());
                    // Verify attempting to read at the current offset will return the appropriate entry (and not throw).
                    @Cleanup ReadResult truncatedResult = store.read(segmentName, readEntry.getStreamSegmentOffset(), 1, TIMEOUT).join();
                    val first = truncatedResult.next();
                    Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType());
                    // Skip over until the first non-truncated offset.
                    expectedCurrentOffset = Math.max(expectedCurrentOffset, startOffset);
                    continue;
                }
                // Non-truncated entry; do the usual verifications.
                readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                byte[] actualData = new byte[readEntryContents.getLength()];
                StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
                AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
                expectedCurrentOffset += readEntryContents.getLength();
                // Every other read, determine if we should truncate or not.
                if (truncate) {
                    long truncateOffset;
                    if (segmentName.hashCode() % 2 == 0) {
                        // Truncate just beyond the current read offset.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset + 1);
                    } else {
                        // Truncate half of what we read so far.
                        truncateOffset = Math.min(segmentLength, expectedCurrentOffset / 2 + 1);
                    }
                    startOffsets.put(segmentName, truncateOffset);
                    store.truncateStreamSegment(segmentName, truncateOffset, TIMEOUT).join();
                }
                truncate = !truncate;
            }
            Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
        }
    }
}

Example 2 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkReads.

private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        long segmentLength = store.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength);
        long expectedCurrentOffset = 0;
        @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) segmentLength, TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in StreamSegmentContainerTests; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            if (!readEntry.getContent().isDone()) {
                readEntry.requestContent(TIMEOUT);
            }
            readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            ReadResultEntryContents readEntryContents = readEntry.getContent().join();
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
    }
}

Example 3 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class StreamSegmentContainerTests method checkReadIndex.

private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, TestContext context) throws Exception {
    waitForOperationsInReadIndex(context.container);
    for (String segmentName : segmentContents.keySet()) {
        long segmentLength = lengths.get(segmentName);
        long startOffset = truncationOffsets.getOrDefault(segmentName, 0L);
        val si = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset());
        Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength());
        byte[] expectedData = segmentContents.get(segmentName).toByteArray();
        long expectedCurrentOffset = startOffset;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            ReadResultEntryContents readEntryContents = readEntry.getContent().join();
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
    }
}

Example 4 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class StreamSegmentContainerTests method waitForOperationsInReadIndex.

/**
 * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog.
 * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to
 * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use
 * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it
 * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when
 * it is, it is guaranteed that everything prior to that has been committed.
 */
private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    String segmentName = "test" + System.nanoTime();
    container.createStreamSegment(segmentName, null, timer.getRemaining()).thenCompose(v -> container.append(segmentName, new byte[1], null, timer.getRemaining())).thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())).thenCompose(rr -> {
        ReadResultEntry rre = rr.next();
        rre.requestContent(TIMEOUT);
        return rre.getContent().thenRun(rr::close);
    }).thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}

Example 5 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class AsyncReadResultProcessor method processResult.

// endregion
// region Processing
private void processResult(Executor executor) {
    // Process the result, one entry at a time, until one of the stopping conditions occurs.
    AtomicBoolean shouldContinue = new AtomicBoolean(true);
    Futures.loop(() -> !this.closed.get() && shouldContinue.get(), () -> {
        CompletableFuture<ReadResultEntry> resultEntryFuture = fetchNextEntry();
        shouldContinue.set(resultEntryFuture != null);
        return resultEntryFuture != null ? resultEntryFuture : CompletableFuture.completedFuture(null);
    }, resultEntry -> {
        if (resultEntry != null) {
            shouldContinue.set(this.entryHandler.processEntry(resultEntry));
        }
    }, executor).whenComplete(// Make sure always close the result processor when done (with our without failures).
    (r, ex) -> close(ex));
}