Examples with ReaderGroupManager io.pravega.client.admin.ReaderGroupManager used on opensource projects

Example 1 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class EndToEndTxnWithScaleTest method testTxnWithScale.

@Test(timeout = 10000)
public void testTxnWithScale() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutScaleGracePeriod(10000).transactionTimeoutTime(10000).build());
    Transaction<String> transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest1");
    transaction.commit();
    // scale
    Stream stream = new StreamImpl("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executorService).getFuture().get();
    assertTrue(result);
    transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest2");
    transaction.commit();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest1", event.getEvent());
    event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest2", event.getEvent());
}

Example 2 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class BatchClientTest method testBatchClient.

@Test(timeout = 50000)
public void testBatchClient() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    // create reader and writer.
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope(SCOPE, controllerUri);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM)).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, serializer, EventWriterConfig.builder().build());
    // write events to stream with 1 segment.
    writeEvents(writer);
    // scale up and write events.
    Stream stream = new StreamImpl(SCOPE, STREAM);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue("Scale up operation", result);
    writeEvents(writer);
    // scale down and write events.
    map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    result = controller.scaleStream(stream, Arrays.asList(1, 2, 3), map, executor).getFuture().get();
    assertTrue("Scale down operation result", result);
    writeEvents(writer);
    BatchClient batchClient = clientFactory.createBatchClient();
    // List out all the segments in the stream.
    ArrayList<SegmentRange> segments = Lists.newArrayList(batchClient.getSegments(stream, null, null).getIterator());
    assertEquals("Expected number of segments", 6, segments.size());
    // Batch read all events from stream.
    List<String> batchEventList = new ArrayList<>();
    segments.forEach(segInfo -> {
        @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(segInfo, serializer);
        batchEventList.addAll(Lists.newArrayList(segmentIterator));
    });
    assertEquals("Event count", 9, batchEventList.size());
    // read from a given offset.
    Segment seg0 = new Segment(SCOPE, STREAM, 0);
    SegmentRange seg0Info = SegmentRangeImpl.builder().segment(seg0).startOffset(60).endOffset(90).build();
    @Cleanup SegmentIterator<String> seg0Iterator = batchClient.readSegment(seg0Info, serializer);
    ArrayList<String> dataAtOffset = Lists.newArrayList(seg0Iterator);
    assertEquals(1, dataAtOffset.size());
    assertEquals(DATA_OF_SIZE_30, dataAtOffset.get(0));
}

Example 3 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class EventProcessorTest method testEventProcessorGroupRebalance.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testEventProcessorGroupRebalance() throws CheckpointStoreException, ReinitializationRequiredException {
    String systemName = "rebalance";
    String readerGroupName = "rebalance";
    CheckpointStore checkpointStore = spy(CheckpointStoreFactory.createInMemoryStore());
    checkpointStore.addReaderGroup(PROCESS, readerGroupName);
    EventProcessorGroupConfig config = createEventProcessorGroupConfig(2);
    EventStreamClientFactory clientFactory = Mockito.mock(EventStreamClientFactory.class);
    EventStreamReader<TestEvent> reader = Mockito.mock(EventStreamReader.class);
    Mockito.when(reader.readNextEvent(anyLong())).thenReturn(Mockito.mock(EventReadImpl.class));
    Mockito.when(clientFactory.createReader(anyString(), anyString(), any(), any())).thenAnswer(x -> reader);
    Mockito.when(clientFactory.<String>createEventWriter(anyString(), any(), any())).thenReturn(new EventStreamWriterMock<>());
    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    Mockito.when(readerGroupManager.getReaderGroup(anyString())).then(invocation -> readerGroup);
    EventProcessorSystemImpl system = new EventProcessorSystemImpl(systemName, PROCESS, SCOPE, clientFactory, readerGroupManager);
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false)).serializer(new EventSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).minRebalanceIntervalMillis(0L).build();
    // Create EventProcessorGroup.
    @Cleanup EventProcessorGroupImpl<TestEvent> group = (EventProcessorGroupImpl<TestEvent>) system.createEventProcessorGroup(eventProcessorConfig, checkpointStore, executor);
    group.awaitRunning();
    ConcurrentHashMap<String, EventProcessorCell<TestEvent>> eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    List<String> readerIds = eventProcessorMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
    // region case 1: even distribution - 2 readers with 2 segments each
    HashMap<String, Integer> distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 2);
    distribution.put(readerIds.get(1), 2);
    ReaderSegmentDistribution readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(0).build();
    Mockito.when(readerGroup.getReaderSegmentDistribution()).thenReturn(readerSegmentDistribution);
    // call rebalance. no new readers should be added or existing reader removed.
    group.rebalance();
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    // the original readers should not have been replaced
    assertTrue(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
    // endregion
    // region case 2: two external readers with 0 segment assignment and 2 overloaded readers in the
    // readergroup. unassigned = 0
    String reader2 = "reader2";
    String reader3 = "reader3";
    distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 2);
    distribution.put(readerIds.get(1), 2);
    distribution.put(reader2, 0);
    distribution.put(reader3, 0);
    readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(0).build();
    Mockito.when(readerGroup.getReaderSegmentDistribution()).thenReturn(readerSegmentDistribution);
    // call rebalance. this should replace existing overloaded readers
    group.rebalance();
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertFalse(eventProcessorMap.containsKey(readerIds.get(0)));
    assertFalse(eventProcessorMap.containsKey(readerIds.get(1)));
    Enumeration<String> keys = eventProcessorMap.keys();
    String firstReplacement = keys.nextElement();
    String secondReplacement = keys.nextElement();
    // verify that checkpointstore.addreader is called twice
    verify(checkpointStore, times(2)).addReader(any(), any(), eq(firstReplacement));
    verify(checkpointStore, times(2)).addReader(any(), any(), eq(secondReplacement));
    // update the readers in the readergroup
    readerIds = eventProcessorMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
    // endregion
    // region case 3: even distribution among 4 readers
    distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 1);
    distribution.put(readerIds.get(1), 1);
    distribution.put(reader2, 1);
    distribution.put(reader3, 1);
    readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(0).build();
    Mockito.when(readerGroup.getReaderSegmentDistribution()).thenReturn(readerSegmentDistribution);
    // call rebalance. nothing should happen
    group.rebalance();
    // no change to the group
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertTrue(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
    // endregion
    // region case 4: with 1 overloaded reader and 2 unassigned segments
    distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 2);
    distribution.put(readerIds.get(1), 0);
    distribution.put(reader2, 0);
    distribution.put(reader3, 0);
    readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(2).build();
    Mockito.when(readerGroup.getReaderSegmentDistribution()).thenReturn(readerSegmentDistribution);
    // call rebalance. overloaded reader should be replaced
    group.rebalance();
    // reader0 should have been replaced.
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertFalse(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
    // endregion
    readerIds = eventProcessorMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
    distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 2);
    distribution.put(readerIds.get(1), 0);
    distribution.put(reader2, 0);
    distribution.put(reader3, 0);
    readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(2).build();
    // case 5: region failure cases
    doThrow(new RuntimeException("reader group throws")).when(readerGroup).getReaderSegmentDistribution();
    // exception should be handled and there should be no state change in event processor
    group.rebalance();
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertTrue(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
    // now reset the distribution
    doReturn(readerSegmentDistribution).when(readerGroup).getReaderSegmentDistribution();
    // throw from checkpoint store
    doThrow(new CheckpointStoreException("checkpoint store exception")).when(checkpointStore).addReader(anyString(), anyString(), anyString());
    // exception should have been thrown and handled
    group.rebalance();
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertTrue(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
    // endregion
    // Stop the group, and await its termmination.
    group.stopAsync();
    group.awaitTerminated();
    // call rebalance after shutdown such that replace cell is called - this should throw precondition failed exception
    readerIds = eventProcessorMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
    distribution = new HashMap<>();
    distribution.put(readerIds.get(0), 2);
    distribution.put(readerIds.get(1), 2);
    distribution.put(reader2, 0);
    distribution.put(reader3, 0);
    readerSegmentDistribution = ReaderSegmentDistribution.builder().readerSegmentDistribution(distribution).unassignedSegments(0).build();
    Mockito.when(readerGroup.getReaderSegmentDistribution()).thenReturn(readerSegmentDistribution);
    // calling rebalance on terminated group will result in Precondition failure with exception logged and ignored
    // and no rebalance occurring.
    // exception should have been thrown and handled
    group.rebalance();
    eventProcessorMap = group.getEventProcessorMap();
    assertEquals(2, eventProcessorMap.size());
    assertTrue(eventProcessorMap.containsKey(readerIds.get(0)));
    assertTrue(eventProcessorMap.containsKey(readerIds.get(1)));
// endregion
}

Example 4 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class EventProcessorTest method createMockSystem.

private EventProcessorSystemImpl createMockSystem(final String name, final String processId, final String scope, final SequenceAnswer<EventStreamReader<TestEvent>> readers, final EventStreamWriter<TestEvent> writer, final String readerGroupName) {
    EventStreamClientFactory clientFactory = Mockito.mock(EventStreamClientFactory.class);
    Mockito.when(clientFactory.createReader(anyString(), anyString(), any(), any())).thenAnswer(readers);
    Mockito.when(clientFactory.<TestEvent>createEventWriter(anyString(), any(), any())).thenReturn(writer);
    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    Mockito.when(readerGroupManager.getReaderGroup(anyString())).then(invocation -> readerGroup);
    return new EventProcessorSystemImpl(name, processId, scope, clientFactory, readerGroupManager);
}

Example 5 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class ReadWriteTest method readWriteTest.

@Test(timeout = 60000)
public void readWriteTest() throws InterruptedException, ExecutionException {
    String scope = "testMultiReaderWriterScope";
    String readerGroupName = "testMultiReaderWriterReaderGroup";
    // 20  readers -> 20 stream segments ( to have max read parallelism)
    ScalingPolicy scalingPolicy = ScalingPolicy.fixed(20);
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(scalingPolicy).build();
    eventsReadFromPravega = new ConcurrentLinkedQueue<>();
    // data used by each of the writers.
    eventData = new AtomicLong();
    // used by readers to maintain a count of events.
    eventReadCount = new AtomicLong();
    stopReadFlag = new AtomicBoolean(false);
    ClientConfig clientConfig = ClientConfig.builder().build();
    try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
        StreamManager streamManager = new StreamManagerImpl(controller, cp)) {
        // create a scope
        Boolean createScopeStatus = streamManager.createScope(scope);
        log.info("Create scope status {}", createScopeStatus);
        // create a stream
        Boolean createStreamStatus = streamManager.createStream(scope, STREAM_NAME, config);
        log.info("Create stream status {}", createStreamStatus);
    }
    try (ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory)) {
        // start writing events to the stream
        log.info("Creating {} writers", NUM_WRITERS);
        List<CompletableFuture<Void>> writerList = new ArrayList<>();
        for (int i = 0; i < NUM_WRITERS; i++) {
            log.info("Starting writer{}", i);
            writerList.add(startNewWriter(eventData, clientFactory));
        }
        // create a reader group
        log.info("Creating Reader group : {}", readerGroupName);
        readerGroupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).build());
        log.info("Reader group name {} ", readerGroupManager.getReaderGroup(readerGroupName).getGroupName());
        log.info("Reader group scope {}", readerGroupManager.getReaderGroup(readerGroupName).getScope());
        // create readers
        log.info("Creating {} readers", NUM_READERS);
        List<CompletableFuture<Void>> readerList = new ArrayList<>();
        String readerName = "reader" + RandomFactory.create().nextInt(Integer.MAX_VALUE);
        // start reading events
        for (int i = 0; i < NUM_READERS; i++) {
            log.info("Starting reader{}", i);
            readerList.add(startNewReader(readerName + i, clientFactory, readerGroupName, eventsReadFromPravega, eventData, eventReadCount, stopReadFlag));
        }
        // wait for writers completion
        Futures.allOf(writerList).get();
        ExecutorServiceHelpers.shutdown(writerPool);
        // set stop read flag to true
        stopReadFlag.set(true);
        // wait for readers completion
        Futures.allOf(readerList).get();
        ExecutorServiceHelpers.shutdown(readerPool);
        // delete readergroup
        log.info("Deleting readergroup {}", readerGroupName);
        readerGroupManager.deleteReaderGroup(readerGroupName);
    }
    log.info("All writers have stopped. Setting Stop_Read_Flag. Event Written Count:{}, Event Read " + "Count: {}", eventData.get(), eventsReadFromPravega.size());
    assertEquals(TOTAL_NUM_EVENTS, eventsReadFromPravega.size());
    // check unique events.
    assertEquals(TOTAL_NUM_EVENTS, new TreeSet<>(eventsReadFromPravega).size());
    // seal the stream
    CompletableFuture<Boolean> sealStreamStatus = controller.sealStream(scope, STREAM_NAME);
    log.info("Sealing stream {}", STREAM_NAME);
    assertTrue(sealStreamStatus.get());
    // delete the stream
    CompletableFuture<Boolean> deleteStreamStatus = controller.deleteStream(scope, STREAM_NAME);
    log.info("Deleting stream {}", STREAM_NAME);
    assertTrue(deleteStreamStatus.get());
    // delete the  scope
    CompletableFuture<Boolean> deleteScopeStatus = controller.deleteScope(scope);
    log.info("Deleting scope {}", scope);
    assertTrue(deleteScopeStatus.get());
    log.info("Read write test succeeds");
}