Search in sources :

Example 1 with ReadResultEntryContents

use of io.pravega.segmentstore.contracts.ReadResultEntryContents in project pravega by pravega.

the class StreamSegmentContainerTests method checkReadIndex.

private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, TestContext context) throws Exception {
    for (String segmentName : segmentContents.keySet()) {
        long segmentLength = lengths.get(segmentName);
        long startOffset = truncationOffsets.getOrDefault(segmentName, 0L);
        val si = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset());
        Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength());
        byte[] expectedData = segmentContents.get(segmentName).toByteArray();
        long expectedCurrentOffset = startOffset;
        @Cleanup ReadResult readResult =, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry =;
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            ReadResultEntryContents readEntryContents = readEntry.getContent().join();
            byte[] actualData = new byte[readEntryContents.getLength()];
            StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length);
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
Also used : lombok.val(lombok.val) ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) ReadResult(io.pravega.segmentstore.contracts.ReadResult) Cleanup(lombok.Cleanup)

Example 2 with ReadResultEntryContents

use of io.pravega.segmentstore.contracts.ReadResultEntryContents in project pravega by pravega.

the class PravegaRequestProcessor method copyData.

 * Copy all of the contents provided into a byteBuffer and return it.
private ByteBuffer copyData(List<ReadResultEntryContents> contents) {
    int totalSize =;
    ByteBuffer data = ByteBuffer.allocate(totalSize);
    int bytesCopied = 0;
    for (ReadResultEntryContents content : contents) {
        int copied = StreamHelpers.readAll(content.getData(), data.array(), bytesCopied, totalSize - bytesCopied);
        Preconditions.checkState(copied == content.getLength(), "Read fewer bytes than available.");
        bytesCopied += copied;
    return data;
Also used : ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents) ByteBuffer(java.nio.ByteBuffer) SneakyThrows(lombok.SneakyThrows)

Example 3 with ReadResultEntryContents

use of io.pravega.segmentstore.contracts.ReadResultEntryContents in project pravega by pravega.

the class StreamSegmentReadIndex method triggerFutureReads.

// endregion
// region Reading
 * Triggers all future reads that have a starting offset before the given value.
 * @throws IllegalStateException If the read index is in recovery mode.
void triggerFutureReads() {
    Exceptions.checkNotClosed(this.closed, this);
    Preconditions.checkState(!this.recoveryMode, "StreamSegmentReadIndex is in Recovery Mode.");
    // Get all eligible Future Reads which wait for data prior to the end offset.
    // Since we are not actually using this entry's data, there is no need to 'touch' it.
    ReadIndexEntry lastEntry;
    synchronized (this.lock) {
        lastEntry = this.indexEntries.getLast();
    if (lastEntry == null) {
        // Nothing to do.
    Collection<FutureReadResultEntry> futureReads;
    boolean sealed = this.metadata.isSealed();
    if (sealed) {
        // Get everything, even if some Future Reads are in the future - those will eventually return EndOfSegment.
        futureReads = this.futureReads.pollAll();
    } else {
        // Get only those up to the last offset of the last append.
        futureReads = this.futureReads.poll(lastEntry.getLastStreamSegmentOffset());
    log.debug("{}: triggerFutureReads (Count = {}, Offset = {}, Sealed = {}).", this.traceObjectId, futureReads.size(), lastEntry.getLastStreamSegmentOffset(), sealed);
    for (FutureReadResultEntry r : futureReads) {
        ReadResultEntry entry = getSingleReadResultEntry(r.getStreamSegmentOffset(), r.getRequestedReadLength());
        assert entry != null : "Serving a StorageReadResultEntry with a null result";
        assert !(entry instanceof FutureReadResultEntry) : "Serving a FutureReadResultEntry with another FutureReadResultEntry.";
        log.trace("{}: triggerFutureReads (Offset = {}, Type = {}).", this.traceObjectId, r.getStreamSegmentOffset(), entry.getType());
        if (entry.getType() == ReadResultEntryType.EndOfStreamSegment) {
            // We have attempted to read beyond the end of the stream. Fail the read request with the appropriate message.
   StreamSegmentSealedException(String.format("StreamSegment has been sealed at offset %d. There can be no more reads beyond this offset.", this.metadata.getLength())));
        } else {
            if (!entry.getContent().isDone()) {
                // Normally, all Future Reads are served from Cache, since they reflect data that has just been appended.
                // However, it's possible that after recovery, we get a read for some data that we do not have in the
                // cache (but it's not a tail read) - this data exists in Storage but our StorageLength has not yet been
                // updated. As such, the only solution we have is to return a FutureRead which will be satisfied when
                // the Writer updates the StorageLength (and trigger future reads). In that scenario, entry we get
                // will likely not be auto-fetched, so we need to request the content.
            CompletableFuture<ReadResultEntryContents> entryContent = entry.getContent();
            Futures.exceptionListener(entryContent, r::fail);
Also used : Consumer(java.util.function.Consumer) ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry)

Example 4 with ReadResultEntryContents

use of io.pravega.segmentstore.contracts.ReadResultEntryContents in project pravega by pravega.

the class PravegaRequestProcessor method collectCachedEntries.

 * Reads all of the cachedEntries from the ReadResult and puts their content into the cachedEntries list.
 * Upon encountering a non-cached entry, it stops iterating and returns it.
private ReadResultEntry collectCachedEntries(long initialOffset, ReadResult readResult, ArrayList<ReadResultEntryContents> cachedEntries) {
    long expectedOffset = initialOffset;
    while (readResult.hasNext()) {
        ReadResultEntry entry =;
        if (entry.getType() == Cache) {
            Preconditions.checkState(entry.getStreamSegmentOffset() == expectedOffset, "Data returned from read was not contiguous.");
            ReadResultEntryContents content = entry.getContent().getNow(null);
            expectedOffset += content.getLength();
        } else {
            return entry;
    return null;
Also used : ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry)

Example 5 with ReadResultEntryContents

use of io.pravega.segmentstore.contracts.ReadResultEntryContents in project pravega by pravega.

the class TestReadResultHandler method processEntry.

public boolean processEntry(ReadResultEntry e) {
    ReadResultEntryContents c = e.getContent().join();
    byte[] data = new byte[c.getLength()];
    try {
        StreamHelpers.readAll(c.getData(), data, 0, data.length);
        return true;
    } catch (Exception ex) {
        return false;
Also used : ReadResultEntryContents(io.pravega.segmentstore.contracts.ReadResultEntryContents)


ReadResultEntryContents (io.pravega.segmentstore.contracts.ReadResultEntryContents)5 ReadResultEntry (io.pravega.segmentstore.contracts.ReadResultEntry)3 ReadResult (io.pravega.segmentstore.contracts.ReadResult)1 StreamSegmentSealedException (io.pravega.segmentstore.contracts.StreamSegmentSealedException)1 ByteBuffer (java.nio.ByteBuffer)1 Consumer (java.util.function.Consumer)1 Cleanup (lombok.Cleanup)1 SneakyThrows (lombok.SneakyThrows)1 lombok.val (lombok.val)1