Examples with EventSegmentReader io.pravega.client.segment.impl.EventSegmentReader used on opensource projects

Example 6 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testSegmentSplit.

@Test(timeout = 10000)
public void testSegmentSplit() throws EndOfSegmentException, SegmentTruncatedException, SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    // Prep the mocks.
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    // Mock for the two SegmentInputStreams.
    Segment segment1 = Segment.fromScopedName("Foo/Bar/1");
    SegmentWithRange s1range = new SegmentWithRange(segment1, 0, 1);
    EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentInputStream1.isSegmentReady()).thenReturn(true);
    Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OF_SEGMENT_REACHED));
    Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment1);
    Segment segment2 = Segment.fromScopedName("Foo/Bar/2");
    SegmentWithRange s2range = new SegmentWithRange(segment2, 0, 0.5);
    EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(writeInt(stream2, 2));
    Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(segment2);
    Segment segment3 = Segment.fromScopedName("Foo/Bar/3");
    SegmentWithRange s3range = new SegmentWithRange(segment3, 0.5, 1.0);
    EventSegmentReader segmentInputStream3 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream3 = segmentStreamFactory.createOutputStreamForSegment(segment3, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    Mockito.when(segmentInputStream3.read(anyLong())).thenReturn(writeInt(stream3, 3));
    Mockito.when(segmentInputStream3.getSegmentId()).thenReturn(segment3);
    SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment1), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream1);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment2), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream2);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment3), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream3);
    Mockito.when(groupState.getEndOffsetForSegment(any())).thenReturn(Long.MAX_VALUE);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(anyLong(), any())).thenReturn(ImmutableMap.of(s1range, 0L)).thenReturn(Collections.emptyMap());
    InOrder inOrder = Mockito.inOrder(segmentInputStream1, groupState);
    EventRead<byte[]> event = reader.readNextEvent(100L);
    assertNull(event.getEvent());
    event = reader.readNextEvent(100L);
    assertNull(event.getEvent());
    Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint").thenReturn(null);
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(anyLong(), any())).thenReturn(ImmutableMap.of(s2range, 0L, s3range, 0L)).thenReturn(Collections.emptyMap());
    assertEquals("checkpoint", reader.readNextEvent(0).getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Ensure groupstate is not updated before the checkpoint.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(s1range);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    // Now it is called.
    inOrder.verify(groupState, Mockito.times(1)).handleEndOfSegment(s1range);
    assertEquals(ImmutableList.of(segmentInputStream2, segmentInputStream3), reader.getReaders());
}

Example 7 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testTruncatedSegmentDeleted.

/**
 * This test tests a scenario where a lagging reader's SegmentInputStream receives a SegmentTruncatedException and
 * when the reader tries to fetch the start offset of this segment the SegmentStore returns a NoSuchSegmentException.
 */
@Test
public void testTruncatedSegmentDeleted() throws Exception {
    AtomicLong clock = new AtomicLong();
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    SegmentWithRange rangedSegment = new SegmentWithRange(segment, 0, 1);
    // Setup mock.
    SegmentInputStreamFactory segInputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    SegmentMetadataClientFactory segmentMetadataClientFactory = Mockito.mock(SegmentMetadataClientFactory.class);
    SegmentMetadataClient metadataClient = Mockito.mock(SegmentMetadataClient.class);
    EventSegmentReader segmentInputStream = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentMetadataClientFactory.createSegmentMetadataClient(any(Segment.class), any())).thenReturn(metadataClient);
    Mockito.when(segmentInputStream.getSegmentId()).thenReturn(segment);
    Mockito.when(segInputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream);
    // Ensure segmentInputStream.read() returns SegmentTruncatedException.
    Mockito.when(segmentInputStream.isSegmentReady()).thenReturn(true);
    Mockito.when(segmentInputStream.read(anyLong())).thenThrow(SegmentTruncatedException.class);
    // Ensure SegmentInfo returns NoSuchSegmentException.
    Mockito.when(metadataClient.getSegmentInfo()).thenThrow(NoSuchSegmentException.class);
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segInputStreamFactory, segmentMetadataClientFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(rangedSegment, 0L)).thenReturn(Collections.emptyMap());
    InOrder inOrder = Mockito.inOrder(groupState, segmentInputStream);
    // Validate that TruncatedDataException is thrown.
    AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
    inOrder.verify(groupState).getCheckpoint();
    // Ensure this segment is closed.
    inOrder.verify(segmentInputStream, Mockito.times(1)).close();
    // Ensure groupstate is not updated before the checkpoint.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
    Mockito.when(groupState.getCheckpoint()).thenReturn("Foo").thenReturn(null);
    EventRead<byte[]> event = reader.readNextEvent(0);
    assertTrue(event.isCheckpoint());
    assertEquals("Foo", event.getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    assertNull(event.getEvent());
    // Now it is called.
    inOrder.verify(groupState).handleEndOfSegment(rangedSegment);
}

Example 8 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testAcquireSegment.

@SuppressWarnings("unused")
@Test(timeout = 10000)
public void testAcquireSegment() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment1 = Segment.fromScopedName("Foo/Bar/0");
    Segment segment2 = Segment.fromScopedName("Foo/Bar/1");
    Mockito.when(groupState.canAcquireSegmentIfNeeded()).thenReturn(true);
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment1, 0, 0.5), 0L)).thenReturn(ImmutableMap.of(new SegmentWithRange(segment2, 0.5, 1.0), 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    writeInt(stream1, 2);
    writeInt(stream2, 3);
    writeInt(stream2, 4);
    reader.readNextEvent(0);
    List<EventSegmentReader> readers = reader.getReaders();
    assertEquals(1, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    reader.readNextEvent(0);
    readers = reader.getReaders();
    assertEquals(2, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    Assert.assertEquals(segment2, readers.get(1).getSegmentId());
    reader.close();
}

Example 9 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class EventStreamReaderTest method testAcquireSealedSegment.

@Test(timeout = 10000)
public void testAcquireSealedSegment() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment1 = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment1, 0, 0.5), -1L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    writeInt(stream1, 2);
    reader.readNextEvent(0);
    List<EventSegmentReader> readers = reader.getReaders();
    assertEquals(0, readers.size());
    reader.readNextEvent(0);
    readers = reader.getReaders();
    assertEquals(0, readers.size());
    reader.close();
}

Example 10 with EventSegmentReader

use of io.pravega.client.segment.impl.EventSegmentReader in project pravega by pravega.

the class ClientFactoryImpl method createRevisionedStreamClient.

private <T> RevisionedStreamClient<T> createRevisionedStreamClient(Segment segment, Serializer<T> serializer, SynchronizerConfig config) {
    EventSegmentReader in = inFactory.createEventReaderForSegment(segment, config.getReadBufferSize());
    DelegationTokenProvider delegationTokenProvider = DelegationTokenProviderFactory.create(controller, segment, AccessOperation.READ_WRITE);
    ConditionalOutputStream cond = condFactory.createConditionalOutputStream(segment, delegationTokenProvider, config.getEventWriterConfig());
    SegmentMetadataClient meta = metaFactory.createSegmentMetadataClient(segment, delegationTokenProvider);
    return new RevisionedStreamClientImpl<>(segment, in, outFactory, cond, meta, serializer, config.getEventWriterConfig(), delegationTokenProvider);
}