Examples with EventSegmentReader io.pravega.client.segment.impl.EventSegmentReader used on opensource projects

Example 11 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testReadWithEndOfSegmentException.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReadWithEndOfSegmentException() throws Exception {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    // Prep the mocks.
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    // Mock for the two SegmentInputStreams.
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OFFSET_REACHED));
    Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment);
    EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer);
    Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0"));
    Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
    SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream1);
    // Mock Orderer
    Orderer orderer = Mockito.mock(Orderer.class);
    Mockito.when(orderer.nextSegment(any(List.class))).thenReturn(segmentInputStream1).thenReturn(segmentInputStream2);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    InOrder inOrder = Mockito.inOrder(segmentInputStream1, groupState);
    EventRead<byte[]> event = reader.readNextEvent(100L);
    assertNotNull(event.getEvent());
    // Validate that segmentInputStream1.close() is invoked on reaching endOffset.
    Mockito.verify(segmentInputStream1, Mockito.times(1)).close();
    SegmentWithRange segmentWithRange = new SegmentWithRange(segment, 0, 1);
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint").thenReturn(null);
    assertEquals("checkpoint", reader.readNextEvent(0).getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    // Now it is called.
    inOrder.verify(groupState, Mockito.times(1)).handleEndOfSegment(segmentWithRange);
}

Example 12 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testReadWithSegmentTruncatedException.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReadWithSegmentTruncatedException() throws Exception {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    // Prep the mocks.
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    // Mock for the two SegmentInputStreams.
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new SegmentTruncatedException());
    Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment);
    EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer);
    Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0"));
    Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
    SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream1);
    // Mock Orderer
    Orderer orderer = Mockito.mock(Orderer.class);
    Mockito.when(orderer.nextSegment(any(List.class))).thenReturn(segmentInputStream1).thenReturn(segmentInputStream2);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(100L));
}

Example 13 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderImpl method acquireSegmentsIfNeeded.

@GuardedBy("readers")
private boolean acquireSegmentsIfNeeded(PositionInternal position) throws ReaderNotInReaderGroupException {
    Map<SegmentWithRange, Long> newSegments = groupState.acquireNewSegmentsIfNeeded(getLag(), position);
    if (!newSegments.isEmpty()) {
        log.info("{} acquiring segments {}", this, newSegments);
        for (Entry<SegmentWithRange, Long> newSegment : newSegments.entrySet()) {
            long endOffset = groupState.getEndOffsetForSegment(newSegment.getKey().getSegment());
            if (newSegment.getValue() < 0 || (newSegment.getValue() == endOffset && endOffset != Long.MAX_VALUE)) {
                sealedSegments.put(newSegment.getKey().getSegment(), newSegment.getValue());
                ranges.put(newSegment.getKey().getSegment(), newSegment.getKey().getRange());
            } else {
                Segment segment = newSegment.getKey().getSegment();
                EventSegmentReader in = inputStreamFactory.createEventReaderForSegment(segment, config.getBufferSize(), segmentsWithData, endOffset);
                in.setOffset(newSegment.getValue());
                readers.add(in);
                ranges.put(segment, newSegment.getKey().getRange());
            }
        }
        segmentsWithData.release();
        return true;
    }
    return false;
}

Example 14 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderImpl method readNextEventInternal.

private EventRead<Type> readNextEventInternal(long timeoutMillis) throws ReaderNotInReaderGroupException, TruncatedDataException {
    long firstByteTimeoutMillis = Math.min(timeoutMillis, BASE_READER_WAITING_TIME_MS);
    Timer timer = new Timer();
    Segment segment = null;
    long offset = -1;
    ByteBuffer buffer = null;
    do {
        String checkpoint = updateGroupStateIfNeeded();
        if (checkpoint != null) {
            // return checkpoint event to user
            return createEmptyEvent(checkpoint);
        }
        EventSegmentReader segmentReader = orderer.nextSegment(readers);
        if (segmentReader == null) {
            blockFor(firstByteTimeoutMillis);
            segmentsWithData.drainPermits();
            buffer = null;
        } else {
            segment = segmentReader.getSegmentId();
            offset = segmentReader.getOffset();
            try {
                buffer = segmentReader.read(firstByteTimeoutMillis);
            } catch (EndOfSegmentException e) {
                boolean isSegmentSealed = e.getErrorType().equals(END_OF_SEGMENT_REACHED);
                handleEndOfSegment(segmentReader, isSegmentSealed);
                buffer = null;
            } catch (SegmentTruncatedException e) {
                handleSegmentTruncated(segmentReader);
                buffer = null;
            } finally {
                if (buffer == null) {
                    refreshAndGetPosition();
                }
            }
        }
    } while (buffer == null && timer.getElapsedMillis() < timeoutMillis);
    if (buffer == null) {
        log.debug("Empty event returned for reader {} ", groupState.getReaderId());
        return createEmptyEvent(null);
    }
    lastRead = Sequence.create(segment.getSegmentId(), offset);
    int length = buffer.remaining() + WireCommands.TYPE_PLUS_LENGTH_SIZE;
    addSegmentOffsetUpdateIfNeeded(segment, offset + length);
    return new EventReadImpl<>(deserializer.deserialize(buffer), getCurrentPosition(), new EventPointerImpl(segment, offset, length), null);
}

Example 15 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderImpl method fetchEvent.

@Override
public Type fetchEvent(EventPointer pointer) throws NoSuchEventException {
    Preconditions.checkNotNull(pointer);
    // Create SegmentInputStream
    @Cleanup EventSegmentReader inputStream = inputStreamFactory.createEventReaderForSegment(pointer.asImpl().getSegment(), pointer.asImpl().getEventLength());
    inputStream.setOffset(pointer.asImpl().getEventStartOffset());
    // Read event
    try {
        ByteBuffer buffer = inputStream.read();
        return deserializer.deserialize(buffer);
    } catch (EndOfSegmentException e) {
        throw new NoSuchEventException(e.getMessage());
    } catch (NoSuchSegmentException | SegmentTruncatedException e) {
        throw new NoSuchEventException("Event no longer exists.");
    }
}