Examples with ReaderGroupManagerImpl io.pravega.client.admin.impl.ReaderGroupManagerImpl used on opensource projects

Example 36 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndCBRTest method testReaderGroupManualRetention.

@Test(timeout = 60000)
public void testReaderGroupManualRetention() throws Exception {
    String scope = "test";
    String streamName = "testReaderGroupManualRetention";
    String groupName = "testReaderGroupManualRetention-group";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).retentionPolicy(RetentionPolicy.bySizeBytes(10, Long.MAX_VALUE)).build();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).get();
    controller.createStream(scope, streamName, config).get();
    Stream stream = Stream.of(scope, streamName);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    // write events
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    writer.writeEvent("1", "e1").join();
    writer.writeEvent("2", "e2").join();
    // Create a ReaderGroup
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).stream(stream).build());
    // Create a Reader
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("reader1", groupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    EventRead<String> read = reader.readNextEvent(60000);
    assertEquals("e1", read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals("e2", read.getEvent());
    ReaderGroup readerGroup = groupManager.getReaderGroup(groupName);
    Map<Segment, Long> segmentMap = new HashMap<>();
    segmentMap.put(new Segment(scope, streamName, 0), 17L);
    Map<Stream, StreamCut> scResult2 = new HashMap<>();
    scResult2.put(stream, new StreamCutImpl(stream, segmentMap));
    readerGroup.updateRetentionStreamCut(scResult2);
    AssertExtensions.assertEventuallyEquals(true, () -> controller.getSegmentsAtTime(stream, 0L).join().values().stream().anyMatch(off -> off > 0), 30 * 1000L);
    String group2 = groupName + "2";
    groupManager.createReaderGroup(group2, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(NameUtils.getScopedStreamName(scope, streamName)).build());
    EventStreamReader<String> reader2 = clientFactory.createReader("reader2", group2, serializer, ReaderConfig.builder().build());
    EventRead<String> eventRead2 = reader2.readNextEvent(10000);
    assertEquals("e2", eventRead2.getEvent());
}

Example 37 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndReaderGroupTest method testResetSubscriberToNonSubscriberReaderGroup.

@Test(timeout = 30000)
public void testResetSubscriberToNonSubscriberReaderGroup() throws InterruptedException, ExecutionException {
    StreamConfiguration config = getStreamConfig();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    String streamName = "testResetSubscriberToNonSubscriberReaderGroup";
    controller.createScope("test").get();
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    // Create a ReaderGroup
    String group = "testResetSubscriberToNonSubscriberReaderGroup-group";
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).build());
    List<String> subs = controller.listSubscribers("test", streamName).get();
    assertTrue("Subscriber list does not contain required reader group", subs.contains("test/" + group));
    ReaderGroup subGroup = groupManager.getReaderGroup(group);
    subGroup.resetReaderGroup(ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).build());
    subs = controller.listSubscribers("test", streamName).get();
    assertFalse("Subscriber list contains required reader group", subs.contains("test/" + group));
}

Example 38 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndReaderGroupTest method testCreateSubscriberReaderGroup.

@Test(timeout = 30000)
public void testCreateSubscriberReaderGroup() throws InterruptedException, ExecutionException {
    StreamConfiguration config = getStreamConfig();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    String streamName = "testCreateSubscriberReaderGroup";
    controller.createScope("test").get();
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    // Create a ReaderGroup
    String groupName = "testCreateSubscriberReaderGroup-group";
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).build());
    List<String> subs = controller.listSubscribers("test", streamName).get();
    assertTrue("Subscriber list does not contain required reader group", subs.contains("test/" + groupName));
}

Example 39 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndTruncationTest method testSegmentTruncationWhileReading.

/**
 * This test checks the behavior of a reader (or group of readers) based on whether segment truncation takes place
 * while reading (first part of the test) or before starting reading (second part).
 *
 * @throws InterruptedException If the current thread is interrupted while waiting for the Controller service.
 */
@Test(timeout = 60000)
public void testSegmentTruncationWhileReading() throws InterruptedException {
    final int totalEvents = 100;
    final String scope = "truncationTests";
    final String streamName = "testSegmentTruncationWhileReading";
    final String readerGroupName = "RGTestSegmentTruncationWhileReading";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, config).join();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    // Write half of totalEvents to the Stream.
    writeEvents(clientFactory, streamName, totalEvents / 2);
    // Seal current segment (0) and split it into two segments (1,2).
    Stream stream = new StreamImpl(scope, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    assertTrue(controller.scaleStream(stream, Lists.newArrayList(0L), map, executorService()).getFuture().join());
    long one = computeSegmentId(1, 1);
    long two = computeSegmentId(2, 1);
    // Write rest of events to the new Stream segments.
    ReadWriteUtils.writeEvents(clientFactory, streamName, totalEvents, totalEvents / 2);
    // Instantiate readers to consume from Stream.
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
    groupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(100).stream(Stream.of(scope, streamName)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(String.valueOf(0), readerGroupName, new UTF8StringSerializer(), ReaderConfig.builder().build());
    int read = 0;
    while (read < 75) {
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.getEvent() != null) {
            read++;
        }
    }
    // Let readers to consume some events and truncate segment while readers are consuming events
    Exceptions.handleInterrupted(() -> Thread.sleep(500));
    Map<Long, Long> streamCutPositions = new HashMap<>();
    streamCutPositions.put(one, 0L);
    streamCutPositions.put(two, 0L);
    assertTrue(controller.truncateStream(scope, streamName, streamCutPositions).join());
    // Wait for readers to complete and assert that they have read all the events (totalEvents).
    while (read < totalEvents) {
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.getEvent() != null) {
            read++;
        }
    }
    assertEquals(read, totalEvents);
    assertEquals(null, reader.readNextEvent(0).getEvent());
    // Assert that from the truncation call onwards, the available segments are the ones after scaling.
    List<Long> currentSegments = controller.getCurrentSegments(scope, streamName).join().getSegments().stream().map(Segment::getSegmentId).sorted().collect(toList());
    currentSegments.removeAll(Lists.newArrayList(one, two));
    assertTrue(currentSegments.isEmpty());
    // The new set of readers, should only read the events beyond truncation point (segments 1 and 2).
    final String newReaderGroupName = readerGroupName + "new";
    groupManager.createReaderGroup(newReaderGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
    List<CompletableFuture<Integer>> futures = readEvents(clientFactory, newReaderGroupName, 1);
    Futures.allOf(futures).join();
    assertEquals((int) futures.stream().map(CompletableFuture::join).reduce((a, b) -> a + b).get(), totalEvents / 2);
}

Example 40 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndTruncationTest method testTruncation.

@Test(timeout = 60000)
public void testTruncation() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 2)).build();
    String streamName = "testTruncation";
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    String scope = "test";
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, config);
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "truncationTest1").get();
    // scale
    Stream stream = new StreamImpl("test", streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    Boolean result = controller.scaleStream(stream, Lists.newArrayList(0L, 1L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "truncationTest2").get();
    Map<Long, Long> streamCutPositions = new HashMap<>();
    streamCutPositions.put(computeSegmentId(2, 1), 0L);
    streamCutPositions.put(computeSegmentId(3, 1), 0L);
    streamCutPositions.put(computeSegmentId(4, 1), 0L);
    controller.truncateStream(stream.getScope(), stream.getStreamName(), streamCutPositions).join();
    String group = "testTruncation-group";
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", clientConfig, connectionFactory);
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("truncationTest2", event.getEvent());
    event = reader.readNextEvent(1000);
    assertNull(event.getEvent());
}