use of io.pravega.client.segment.impl.SegmentInputStreamFactory in project pravega by pravega.
the class SegmentIteratorTest method testTimeoutError.
@Test(timeout = 5000)
public void testTimeoutError() throws SegmentTruncatedException, EndOfSegmentException {
Segment segment = new Segment("Scope", "Stream", 1);
int endOffset = 10;
SegmentInputStreamFactory factory = mock(SegmentInputStreamFactory.class);
EventSegmentReader input = mock(EventSegmentReader.class);
when(factory.createEventReaderForSegment(segment, 0, endOffset)).thenReturn(input);
when(input.read()).thenReturn(null).thenReturn(stringSerializer.serialize("s"));
@Cleanup SegmentIteratorImpl<String> iter = new SegmentIteratorImpl<>(factory, segment, stringSerializer, 0, endOffset);
assertEquals("s", iter.next());
verify(input, times(2)).read();
when(input.read()).thenThrow(SegmentTruncatedException.class);
assertThrows(TruncatedDataException.class, () -> iter.next());
}
use of io.pravega.client.segment.impl.SegmentInputStreamFactory in project pravega by pravega.
the class RevisionedStreamClientTest method testTimeoutWithStreamIterator.
@Test
public void testTimeoutWithStreamIterator() throws Exception {
String scope = "scope";
String stream = "stream";
// Setup Environment
PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
JavaSerializer<String> serializer = new JavaSerializer<>();
@Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
@Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
createScopeAndStream(scope, stream, controller);
MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
// Setup mock
SegmentOutputStreamFactory outFactory = mock(SegmentOutputStreamFactory.class);
SegmentOutputStream out = mock(SegmentOutputStream.class);
Segment segment = new Segment(scope, stream, 0);
when(outFactory.createOutputStreamForSegment(eq(segment), any(), any(), any(DelegationTokenProvider.class))).thenReturn(out);
SegmentInputStreamFactory inFactory = mock(SegmentInputStreamFactory.class);
EventSegmentReader in = mock(EventSegmentReader.class);
when(inFactory.createEventReaderForSegment(eq(segment), anyInt())).thenReturn(in);
when(in.read(anyLong())).thenReturn(null).thenReturn(serializer.serialize("testData"));
SegmentMetadataClientFactory metaFactory = mock(SegmentMetadataClientFactory.class);
SegmentMetadataClient metaClient = mock(SegmentMetadataClient.class);
when(metaFactory.createSegmentMetadataClient(any(Segment.class), any(DelegationTokenProvider.class))).thenReturn(metaClient);
when(metaClient.getSegmentInfo()).thenReturn(CompletableFuture.completedFuture(new SegmentInfo(segment, 0, 30, false, System.currentTimeMillis())));
@Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, inFactory, streamFactory, streamFactory, metaFactory);
@Cleanup RevisionedStreamClient<String> client = clientFactory.createRevisionedStreamClient(stream, serializer, SynchronizerConfig.builder().build());
Iterator<Entry<Revision, String>> iterator = client.readFrom(new RevisionImpl(segment, 15, 1));
assertTrue("True is expected since offset is less than end offset", iterator.hasNext());
assertNotNull("Verify the entry is not null", iterator.next());
verify(in, times(2)).read(anyLong());
}
use of io.pravega.client.segment.impl.SegmentInputStreamFactory in project pravega by pravega.
the class EventStreamReaderTest method testSegmentSplit.
@Test(timeout = 10000)
public void testSegmentSplit() throws EndOfSegmentException, SegmentTruncatedException, SegmentSealedException, ReaderNotInReaderGroupException {
AtomicLong clock = new AtomicLong();
MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
// Prep the mocks.
ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
// Mock for the two SegmentInputStreams.
Segment segment1 = Segment.fromScopedName("Foo/Bar/1");
SegmentWithRange s1range = new SegmentWithRange(segment1, 0, 1);
EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
Mockito.when(segmentInputStream1.isSegmentReady()).thenReturn(true);
Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OF_SEGMENT_REACHED));
Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment1);
Segment segment2 = Segment.fromScopedName("Foo/Bar/2");
SegmentWithRange s2range = new SegmentWithRange(segment2, 0, 0.5);
EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
@Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(writeInt(stream2, 2));
Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(segment2);
Segment segment3 = Segment.fromScopedName("Foo/Bar/3");
SegmentWithRange s3range = new SegmentWithRange(segment3, 0.5, 1.0);
EventSegmentReader segmentInputStream3 = Mockito.mock(EventSegmentReader.class);
@Cleanup SegmentOutputStream stream3 = segmentStreamFactory.createOutputStreamForSegment(segment3, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
Mockito.when(segmentInputStream3.read(anyLong())).thenReturn(writeInt(stream3, 3));
Mockito.when(segmentInputStream3.getSegmentId()).thenReturn(segment3);
SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment1), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream1);
Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment2), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream2);
Mockito.when(inputStreamFactory.createEventReaderForSegment(eq(segment3), anyInt(), any(Semaphore.class), eq(Long.MAX_VALUE))).thenReturn(segmentInputStream3);
Mockito.when(groupState.getEndOffsetForSegment(any())).thenReturn(Long.MAX_VALUE);
@Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
Mockito.when(groupState.acquireNewSegmentsIfNeeded(anyLong(), any())).thenReturn(ImmutableMap.of(s1range, 0L)).thenReturn(Collections.emptyMap());
InOrder inOrder = Mockito.inOrder(segmentInputStream1, groupState);
EventRead<byte[]> event = reader.readNextEvent(100L);
assertNull(event.getEvent());
event = reader.readNextEvent(100L);
assertNull(event.getEvent());
Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint").thenReturn(null);
Mockito.when(groupState.acquireNewSegmentsIfNeeded(anyLong(), any())).thenReturn(ImmutableMap.of(s2range, 0L, s3range, 0L)).thenReturn(Collections.emptyMap());
assertEquals("checkpoint", reader.readNextEvent(0).getCheckpointName());
inOrder.verify(groupState).getCheckpoint();
// Ensure groupstate is not updated before the checkpoint.
inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(s1range);
event = reader.readNextEvent(0);
assertFalse(event.isCheckpoint());
// Now it is called.
inOrder.verify(groupState, Mockito.times(1)).handleEndOfSegment(s1range);
assertEquals(ImmutableList.of(segmentInputStream2, segmentInputStream3), reader.getReaders());
}
use of io.pravega.client.segment.impl.SegmentInputStreamFactory in project pravega by pravega.
the class EventStreamReaderTest method testTruncatedSegmentDeleted.
/**
* This test tests a scenario where a lagging reader's SegmentInputStream receives a SegmentTruncatedException and
* when the reader tries to fetch the start offset of this segment the SegmentStore returns a NoSuchSegmentException.
*/
@Test
public void testTruncatedSegmentDeleted() throws Exception {
AtomicLong clock = new AtomicLong();
Segment segment = Segment.fromScopedName("Foo/Bar/0");
SegmentWithRange rangedSegment = new SegmentWithRange(segment, 0, 1);
// Setup mock.
SegmentInputStreamFactory segInputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
SegmentMetadataClientFactory segmentMetadataClientFactory = Mockito.mock(SegmentMetadataClientFactory.class);
SegmentMetadataClient metadataClient = Mockito.mock(SegmentMetadataClient.class);
EventSegmentReader segmentInputStream = Mockito.mock(EventSegmentReader.class);
Mockito.when(segmentMetadataClientFactory.createSegmentMetadataClient(any(Segment.class), any())).thenReturn(metadataClient);
Mockito.when(segmentInputStream.getSegmentId()).thenReturn(segment);
Mockito.when(segInputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream);
// Ensure segmentInputStream.read() returns SegmentTruncatedException.
Mockito.when(segmentInputStream.isSegmentReady()).thenReturn(true);
Mockito.when(segmentInputStream.read(anyLong())).thenThrow(SegmentTruncatedException.class);
// Ensure SegmentInfo returns NoSuchSegmentException.
Mockito.when(metadataClient.getSegmentInfo()).thenThrow(NoSuchSegmentException.class);
Orderer orderer = new Orderer();
ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
@Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segInputStreamFactory, segmentMetadataClientFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(rangedSegment, 0L)).thenReturn(Collections.emptyMap());
InOrder inOrder = Mockito.inOrder(groupState, segmentInputStream);
// Validate that TruncatedDataException is thrown.
AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
inOrder.verify(groupState).getCheckpoint();
// Ensure this segment is closed.
inOrder.verify(segmentInputStream, Mockito.times(1)).close();
// Ensure groupstate is not updated before the checkpoint.
inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
Mockito.when(groupState.getCheckpoint()).thenReturn("Foo").thenReturn(null);
EventRead<byte[]> event = reader.readNextEvent(0);
assertTrue(event.isCheckpoint());
assertEquals("Foo", event.getCheckpointName());
inOrder.verify(groupState).getCheckpoint();
// Verify groupstate is not updated before the checkpoint, but is after.
inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
event = reader.readNextEvent(0);
assertFalse(event.isCheckpoint());
assertNull(event.getEvent());
// Now it is called.
inOrder.verify(groupState).handleEndOfSegment(rangedSegment);
}
use of io.pravega.client.segment.impl.SegmentInputStreamFactory in project pravega by pravega.
the class RevisionedStreamClientTest method testRetryOnTimeout.
@Test
public void testRetryOnTimeout() throws ConnectionFailedException {
String scope = "scope";
String stream = "stream";
Segment segment = new Segment(scope, stream, 0L);
// Setup Environment
PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
// Setup Mocks
JavaSerializer<String> serializer = new JavaSerializer<>();
@Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
@Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
// Setup client connection.
ClientConnection c = mock(ClientConnection.class);
connectionFactory.provideConnection(endpoint, c);
// Create Scope and Stream.
createScopeAndStream(scope, stream, controller);
// Create mock ClientFactory.
SegmentInputStreamFactory segInputFactory = new SegmentInputStreamFactoryImpl(controller, connectionFactory);
SegmentOutputStreamFactory segOutputFactory = mock(SegmentOutputStreamFactory.class);
ConditionalOutputStreamFactory condOutputFactory = new ConditionalOutputStreamFactoryImpl(controller, connectionFactory);
SegmentMetadataClientFactory segMetaFactory = mock(SegmentMetadataClientFactory.class);
SegmentMetadataClient segMetaClient = mock(SegmentMetadataClient.class);
when(segMetaFactory.createSegmentMetadataClient(eq(segment), any(DelegationTokenProvider.class))).thenReturn(segMetaClient);
@Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, segInputFactory, segOutputFactory, condOutputFactory, segMetaFactory);
RevisionedStreamClientImpl<String> client = spy((RevisionedStreamClientImpl<String>) clientFactory.createRevisionedStreamClient(stream, serializer, SynchronizerConfig.builder().build()));
// Override the readTimeout value for RevisionedClient to 1 second.
doReturn(1000L).when(client).getReadTimeout();
// Setup the SegmentMetadataClient mock.
doReturn(CompletableFuture.completedFuture(new SegmentInfo(segment, 0L, 30L, false, 1L))).when(segMetaClient).getSegmentInfo();
// Get the iterator from Revisioned Stream Client.
Iterator<Entry<Revision, String>> iterator = client.readFrom(new RevisionImpl(segment, 15, 1));
// since we are trying to read @ offset 15 and the writeOffset is 30L a true is returned for hasNext().
assertTrue(iterator.hasNext());
// Setup mock to validate a retry.
doNothing().doAnswer(i -> {
WireCommands.ReadSegment request = i.getArgument(0);
ReplyProcessor rp = connectionFactory.getProcessor(endpoint);
WireCommands.Event event = new WireCommands.Event(Unpooled.wrappedBuffer(serializer.serialize("A")));
ByteArrayOutputStream bout = new ByteArrayOutputStream();
event.writeFields(new DataOutputStream(bout));
ByteBuf eventData = Unpooled.wrappedBuffer(bout.toByteArray());
// Invoke Reply processor to simulate a successful read.
rp.process(new WireCommands.SegmentRead(request.getSegment(), 15L, true, true, eventData, request.getRequestId()));
return null;
}).when(c).send(any(WireCommands.ReadSegment.class));
Entry<Revision, String> r = iterator.next();
assertEquals("A", r.getValue());
// Verify retries have been performed.
verify(c, times(3)).send(any(WireCommands.ReadSegment.class));
}
Aggregations