Examples with ReaderGroup io.pravega.client.stream.ReaderGroup used on opensource projects

Example 1 with ReaderGroup

use of io.pravega.client.stream.ReaderGroup in project pravega by pravega.

the class EventProcessorTest method createMockSystem.

@SuppressWarnings("unchecked")
private EventProcessorSystemImpl createMockSystem(final String name, final String processId, final String scope, final SequenceAnswer<EventStreamReader<TestEvent>> readers, final EventStreamWriter<TestEvent> writer, final String readerGroupName) {
    ClientFactory clientFactory = Mockito.mock(ClientFactory.class);
    Mockito.when(clientFactory.createReader(anyString(), anyString(), any(), any())).thenAnswer(readers);
    Mockito.when(clientFactory.<TestEvent>createEventWriter(anyString(), any(), any())).thenReturn(writer);
    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    Mockito.when(readerGroupManager.createReaderGroup(anyString(), any(ReaderGroupConfig.class))).then(invocation -> readerGroup);
    return new EventProcessorSystemImpl(name, processId, scope, clientFactory, readerGroupManager);
}

Example 2 with ReaderGroup

use of io.pravega.client.stream.ReaderGroup in project pravega by pravega.

the class ReaderGroupNotificationTest method testEndOfStreamNotifications.

@Test(timeout = 40000)
public void testEndOfStreamNotifications() throws Exception {
    final String streamName = "stream2";
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    // seal stream
    assertTrue(controller.sealStream(SCOPE, streamName).get());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    // Add segment event listener
    Listener<EndOfDataNotification> l1 = notification -> {
        listenerInvoked.set(true);
        listenerLatch.release();
    };
    readerGroup.getEndOfDataNotifier(executor).registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(10000);
    EventRead<String> event2 = reader1.readNextEvent(10000);
    EventRead<String> event3 = reader1.readNextEvent(10000);
    assertNotNull(event1);
    assertEquals("data1", event1.getEvent());
    assertNotNull(event2);
    assertEquals("data2", event2.getEvent());
    assertNull(event3.getEvent());
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
}

Example 3 with ReaderGroup

use of io.pravega.client.stream.ReaderGroup in project pravega by pravega.

the class ReaderGroupNotificationTest method testSegmentNotifications.

@Test(timeout = 40000)
public void testSegmentNotifications() throws Exception {
    final String streamName = "stream1";
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    // Add segment event listener
    Listener<SegmentNotification> l1 = notification -> {
        listenerInvoked.set(true);
        numberOfReaders.set(notification.getNumOfReaders());
        numberOfSegments.set(notification.getNumOfSegments());
        listenerLatch.release();
    };
    readerGroup.getSegmentNotifier(executor).registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(15000);
    EventRead<String> event2 = reader1.readNextEvent(15000);
    assertNotNull(event1);
    assertEquals("data1", event1.getEvent());
    assertNotNull(event2);
    assertEquals("data2", event2.getEvent());
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
    assertEquals(2, numberOfSegments.get());
    assertEquals(1, numberOfReaders.get());
}

Example 4 with ReaderGroup

use of io.pravega.client.stream.ReaderGroup in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytes.

@Test(timeout = 50000)
public void testUnreadBytes() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("unreadbytes").streamName("unreadbytes").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("unreadbytes").get();
    controller.createStream(config).get();
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("unreadbytes", new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    ReaderGroup readerGroup = groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/unreadbytes").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executor);
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
}

Example 5 with ReaderGroup

use of io.pravega.client.stream.ReaderGroup in project pravega by pravega.

the class EndToEndChannelLeakTest method testDetectChannelLeakMultiReader.

@Test(timeout = 30000)
public void testDetectChannelLeakMultiReader() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    // Create a writer.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM_NAME)).build());
    // create a reader.
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Write an event.
    writer.writeEvent("0", "zero").get();
    // Total 4 sockets are open at this point : Writer has 1 connection to segment 0 of stream +
    // Reader has 3 connections (1 metadata client + 1 Writer to _RGreader/0 + 1 Reader to _RGreader/0)
    assertEquals(4, connectionFactory.getActiveChannelCount());
    int channelCount = 4;
    // Read an event.
    EventRead<String> event = reader1.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("zero", event.getEvent());
    // +1 socket to segment 0 of the stream.
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    // scale
    Stream stream = new StreamImpl(SCOPE, STREAM_NAME);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    // No changes to the channel count.
    assertEquals(channelCount, connectionFactory.getActiveChannelCount());
    // Write more events.
    writer.writeEvent("0", "one").get();
    writer.writeEvent("0", "two").get();
    writer.writeEvent("1", "three").get();
    // 2 new connections(+3 connections to the segments 1,2,3 after scale by the writer,
    // -1 connection to segment 0 which is sealed.)
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 2;
    // Add a new reader
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("readerId2", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Creation of a reader will add 3 more connections details similar to the above comment.
    assertEquals(channelCount + 3, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 3;
    event = reader1.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (-1 since segment 0 of stream is sealed + 2 connections to two segments of stream (there are
    // 2 readers and 3 segments and the reader1 will be assigned 2 segments))
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    event = reader2.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (a new connection to the remaining stream segment)
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
}