Examples with ReadResultEntry io.pravega.segmentstore.contracts.ReadResultEntry used on opensource projects

Example 16 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentSeal.

/**
 * Test the seal operation on StreamSegments. Also tests the behavior of Reads (non-tailing) when encountering
 * the end of a sealed StreamSegment.
 */
@Test
public void testSegmentSeal() throws Exception {
    final int appendsPerSegment = 1;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> appendFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    for (String segmentName : segmentNames) {
        ByteArrayOutputStream segmentStream = new ByteArrayOutputStream();
        segmentContents.put(segmentName, segmentStream);
        for (int i = 0; i < appendsPerSegment; i++) {
            byte[] appendData = getAppendData(segmentName, i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            segmentStream.write(appendData);
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Seal first half of segments.
    ArrayList<CompletableFuture<Long>> sealFutures = new ArrayList<>();
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        sealFutures.add(context.container.sealStreamSegment(segmentNames.get(i), TIMEOUT));
    }
    Futures.allOf(sealFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Check that the segments were properly sealed.
    for (int i = 0; i < segmentNames.size(); i++) {
        String segmentName = segmentNames.get(i);
        boolean expectedSealed = i < segmentNames.size() / 2;
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        if (expectedSealed) {
            Assert.assertTrue("Segment is not sealed when it should be " + segmentName, sp.isSealed());
            Assert.assertEquals("Unexpected result from seal() future for segment " + segmentName, sp.getLength(), (long) sealFutures.get(i).join());
            AssertExtensions.assertThrows("Container allowed appending to a sealed segment " + segmentName, context.container.append(segmentName, "foo".getBytes(), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentSealedException);
        } else {
            Assert.assertFalse("Segment is sealed when it shouldn't be " + segmentName, sp.isSealed());
            // Verify we can still append to these segments.
            byte[] appendData = "foo".getBytes();
            context.container.append(segmentName, appendData, null, TIMEOUT).join();
            segmentContents.get(segmentName).write(appendData);
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
        }
    }
    // 4. Reads (regular reads, not tail reads, and only for the sealed segments).
    waitForOperationsInReadIndex(context.container);
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        String segmentName = segmentNames.get(i);
        long segmentLength = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        // Read starting 1 byte from the end - make sure it wont hang at the end by turning into a future read.
        final int totalReadLength = 1;
        long expectedCurrentOffset = segmentLength - totalReadLength;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, Integer.MAX_VALUE, TIMEOUT).join();
        int readLength = 0;
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            if (readEntry.getStreamSegmentOffset() >= segmentLength) {
                Assert.assertEquals("Unexpected value for isEndOfStreamSegment when reaching the end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                AssertExtensions.assertThrows("ReadResultEntry.getContent() returned a result when reached the end of sealed segment " + segmentName, readEntry::getContent, ex -> ex instanceof IllegalStateException);
            } else {
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment before reaching end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                expectedCurrentOffset += readEntryContents.getLength();
                readLength += readEntryContents.getLength();
            }
        }
        Assert.assertEquals("Unexpected number of bytes read.", totalReadLength, readLength);
        Assert.assertTrue("ReadResult was not closed when reaching the end of sealed segment" + segmentName, readResult.isClosed());
    }
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 17 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegmentTruncated.

@Test(timeout = 20000)
public void testReadSegmentTruncated() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Truncated, 0, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    StreamSegmentInformation info = StreamSegmentInformation.builder().name(streamSegmentName).length(1234).startOffset(123).build();
    when(store.getStreamSegmentInfo(streamSegmentName, false, PravegaRequestProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(info));
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(store).getStreamSegmentInfo(streamSegmentName, false, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentIsTruncated(0, streamSegmentName, info.getStartOffset()));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 18 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegment.

@Test(timeout = 20000)
public void testReadSegment() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Cache, 0, readLength);
    entry1.complete(new ReadResultEntryContents(new ByteArrayInputStream(data), data.length));
    TestReadResultEntry entry2 = new TestReadResultEntry(ReadResultEntryType.Future, data.length, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    results.add(entry2);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, true, false, ByteBuffer.wrap(data)));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    entry2.complete(new ReadResultEntryContents(new ByteArrayInputStream(data), data.length));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 19 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegmentEmptySealed.

@Test(timeout = 20000)
public void testReadSegmentEmptySealed() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.EndOfStreamSegment, 0, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, false, true, ByteBuffer.wrap(new byte[0])));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 20 with ReadResultEntry

use of io.pravega.segmentstore.contracts.ReadResultEntry in project pravega by pravega.

the class ContainerReadIndexTests method testTruncateConcurrently.

/**
 * Tests a scenario of truncation that happens concurrently with reading (segment is truncated while reading).
 */
@Test
public void testTruncateConcurrently() throws Exception {
    @Cleanup TestContext context = new TestContext();
    List<Long> segmentIds = createSegments(context).subList(0, 1);
    long segmentId = segmentIds.get(0);
    ByteArrayOutputStream segmentContents = new ByteArrayOutputStream();
    appendData(segmentIds, Collections.singletonMap(segmentId, segmentContents), context);
    // Begin a read result.
    UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
    @Cleanup ReadResult rr = context.readIndex.read(segmentId, 0, (int) sm.getLength(), TIMEOUT);
    ReadResultEntry firstEntry = rr.next();
    firstEntry.requestContent(TIMEOUT);
    int firstEntryLength = firstEntry.getContent().join().getLength();
    AssertExtensions.assertLessThan("Unexpected length of the first read result entry.", sm.getLength(), firstEntryLength);
    // Truncate the segment just after the end of the first returned read result.
    sm.setStartOffset(firstEntryLength + 1);
    ReadResultEntry secondEntry = rr.next();
    Assert.assertTrue("Unexpected ReadResultEntryType.isTerminal of truncated result entry.", secondEntry.getType().isTerminal());
    Assert.assertEquals("Unexpected ReadResultEntryType of truncated result entry.", ReadResultEntryType.Truncated, secondEntry.getType());
    AssertExtensions.assertThrows("Expecting getContent() to return a failed CompletableFuture.", secondEntry::getContent, ex -> ex instanceof StreamSegmentTruncatedException);
    Assert.assertFalse("Unexpected result from hasNext after processing terminal result entry.", rr.hasNext());
}