Examples with ReaderGroupManager io.pravega.client.admin.ReaderGroupManager used on opensource projects

Example 66 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class ReaderCheckpointTest method readerCheckpointTest.

@Test
public void readerCheckpointTest() {
    controllerURI = fetchControllerURI();
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    StreamManager streamManager = StreamManager.create(clientConfig);
    assertTrue("Creating Scope", streamManager.createScope(SCOPE_1));
    assertTrue("Creating stream", streamManager.createStream(SCOPE_1, STREAM, streamConfig));
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SCOPE_1, clientConfig);
    readerGroupManager.createReaderGroup(READER_GROUP_NAME, ReaderGroupConfig.builder().stream(io.pravega.client.stream.Stream.of(SCOPE_1, STREAM)).build());
    @Cleanup ReaderGroup readerGroup = readerGroupManager.getReaderGroup(READER_GROUP_NAME);
    int startInclusive = 1;
    int endExclusive = 100;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_1, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_1, startInclusive, endExclusive);
    // initiate checkpoint100
    Checkpoint checkPoint100 = createCheckPointAndVerify(readerGroup, "batch100");
    // write and read events 100 to 200
    startInclusive = 100;
    endExclusive = 200;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_1, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_1, startInclusive, endExclusive);
    // reset to check point 100
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(checkPoint100).build());
    readEventsAndVerify(SCOPE_1, 100, endExclusive);
    // initiate checkpoint200
    Checkpoint checkPoint200 = createCheckPointAndVerify(readerGroup, "batch200");
    // write and read events 200 to 300
    startInclusive = 200;
    endExclusive = 300;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_1, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_1, startInclusive, endExclusive);
    // reset back to checkpoint 200
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(checkPoint200).build());
    readEventsAndVerify(SCOPE_1, 200, endExclusive);
    // reset back to checkpoint 100
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(checkPoint100).build());
    readEventsAndVerify(SCOPE_1, 100, endExclusive);
    // clean up
    readerGroupManager.deleteReaderGroup(READER_GROUP_NAME);
}

Example 67 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class ReaderCheckpointTest method generateStreamCutsTest.

@Test
public void generateStreamCutsTest() {
    controllerURI = fetchControllerURI();
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    StreamManager streamManager = StreamManager.create(clientConfig);
    assertTrue("Creating Scope", streamManager.createScope(SCOPE_2));
    assertTrue("Creating stream", streamManager.createStream(SCOPE_2, STREAM, streamConfig));
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SCOPE_2, clientConfig);
    readerGroupManager.createReaderGroup(READER_GROUP_NAME, ReaderGroupConfig.builder().stream(io.pravega.client.stream.Stream.of(SCOPE_2, STREAM)).groupRefreshTimeMillis(GROUP_REFRESH_TIME_MILLIS).build());
    @Cleanup ReaderGroup readerGroup = readerGroupManager.getReaderGroup(READER_GROUP_NAME);
    int startInclusive = 1;
    int endExclusive = 100;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_2, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_2, startInclusive, endExclusive);
    // Obtain StreamCuts at 100th event.
    Map<Stream, StreamCut> cutAt100 = generateStreamCuts(readerGroup);
    // Write and read events 100 to 200
    startInclusive = 100;
    endExclusive = 200;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_2, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_2, startInclusive, endExclusive);
    // Reset to stream cut pointing to 100th event.
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(cutAt100).build());
    readEventsAndVerify(SCOPE_2, 100, endExclusive);
    // Obtain stream cut at 200th event.
    Map<Stream, StreamCut> cutAt200 = generateStreamCuts(readerGroup);
    // Write and read events 200 to 300.
    startInclusive = 200;
    endExclusive = 300;
    log.info("Write events with range [{},{})", startInclusive, endExclusive);
    writeEvents(SCOPE_2, IntStream.range(startInclusive, endExclusive).boxed().collect(Collectors.toList()));
    readEventsAndVerify(SCOPE_2, startInclusive, endExclusive);
    // Reset back to stream cut pointing to 200th event.
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(cutAt200).build());
    readEventsAndVerify(SCOPE_2, 200, endExclusive);
    // Reset back to stream cut pointing to 100th event.
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(cutAt100).build());
    readEventsAndVerify(SCOPE_2, 100, endExclusive);
    // clean up
    readerGroupManager.deleteReaderGroup(READER_GROUP_NAME);
}

Example 68 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class SingleSubscriberUpdateRetentionStreamCutTest method singleSubscriberCBRTest.

@Test
public void singleSubscriberCBRTest() throws Exception {
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, clientConfig);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, new JavaSerializer<>(), EventWriterConfig.builder().build());
    // Write a single event.
    log.info("Writing event e1 to {}/{}", SCOPE, STREAM);
    writer.writeEvent("e1", SIZE_30_EVENT).join();
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    readerGroupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM)).build());
    ReaderGroup readerGroup = readerGroupManager.getReaderGroup(READER_GROUP);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(READER_GROUP + "-" + 1, READER_GROUP, new JavaSerializer<>(), readerConfig);
    // Read one event.
    log.info("Reading event e1 from {}/{}", SCOPE, STREAM);
    EventRead<String> read = reader.readNextEvent(READ_TIMEOUT);
    assertFalse(read.isCheckpoint());
    assertEquals("data of size 30", read.getEvent());
    // Update the retention stream-cut.
    log.info("{} generating stream-cuts for {}/{}", READER_GROUP, SCOPE, STREAM);
    CompletableFuture<Map<Stream, StreamCut>> futureCuts = readerGroup.generateStreamCuts(streamCutExecutor);
    // Wait for 5 seconds to force reader group state update. This will allow for the silent
    // checkpoint event generated as part of generateStreamCuts to be picked and processed.
    Exceptions.handleInterrupted(() -> TimeUnit.SECONDS.sleep(5));
    EventRead<String> emptyEvent = reader.readNextEvent(READ_TIMEOUT);
    assertTrue("Stream-cut generation did not complete", Futures.await(futureCuts, 10_000));
    Map<Stream, StreamCut> streamCuts = futureCuts.join();
    log.info("{} updating its retention stream-cut to {}", READER_GROUP, streamCuts);
    readerGroup.updateRetentionStreamCut(streamCuts);
    // Write two more events.
    log.info("Writing event e2 to {}/{}", SCOPE, STREAM);
    writer.writeEvent("e2", SIZE_30_EVENT).join();
    log.info("Writing event e3 to {}/{}", SCOPE, STREAM);
    writer.writeEvent("e3", SIZE_30_EVENT).join();
    // Check to make sure truncation happened after the first event.
    // The timeout is 5 minutes as the retention period is set to 2 minutes. We allow for 2 cycles to fully complete
    // and a little longer in order to confirm that the retention has taken place.
    AssertExtensions.assertEventuallyEquals("Truncation did not take place at offset 30.", true, () -> controller.getSegmentsAtTime(new StreamImpl(SCOPE, STREAM), 0L).join().values().stream().anyMatch(off -> off >= 30), 1000, 5 * 60 * 1000L);
    // Read next event.
    log.info("Reading event e2 from {}/{}", SCOPE, STREAM);
    read = reader.readNextEvent(READ_TIMEOUT);
    assertFalse(read.isCheckpoint());
    assertEquals("data of size 30", read.getEvent());
    // Update the retention stream-cut.
    log.info("{} generating stream-cuts for {}/{}", READER_GROUP, SCOPE, STREAM);
    CompletableFuture<Map<Stream, StreamCut>> futureCuts2 = readerGroup.generateStreamCuts(streamCutExecutor);
    // Wait for 5 seconds to force reader group state update. This will allow for the silent
    // checkpoint event generated as part of generateStreamCuts to be picked and processed.
    Exceptions.handleInterrupted(() -> TimeUnit.SECONDS.sleep(5));
    EventRead<String> emptyEvent2 = reader.readNextEvent(READ_TIMEOUT);
    assertTrue("Stream-cut generation did not complete", Futures.await(futureCuts2, 10_000));
    Map<Stream, StreamCut> streamCuts2 = futureCuts2.join();
    log.info("{} updating its retention stream-cut to {}", READER_GROUP, streamCuts2);
    readerGroup.updateRetentionStreamCut(streamCuts2);
    // Check to make sure truncation happened after the second event.
    // The timeout is 5 minutes as the retention period is set to 2 minutes. We allow for 2 cycles to fully complete
    // and a little longer in order to confirm that the retention has taken place.
    AssertExtensions.assertEventuallyEquals("Truncation did not take place at offset 60", true, () -> controller.getSegmentsAtTime(new StreamImpl(SCOPE, STREAM), 0L).join().values().stream().anyMatch(off -> off >= 60), 1000, 5 * 60 * 1000L);
}

Example 69 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class PravegaTest method simpleTest.

/**
 * Invoke the simpleTest, ensure we are able to produce  events.
 * The test fails incase of exceptions while writing to the stream.
 */
@Test
public void simpleTest() {
    Service conService = Utils.createPravegaControllerService(null);
    List<URI> ctlURIs = conService.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    log.info("Invoking create stream with Controller URI: {}", controllerUri);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(Utils.buildClientConfig(controllerUri));
    @Cleanup ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(Utils.buildClientConfig(controllerUri)).build(), connectionFactory.getInternalExecutor());
    assertTrue(controller.createScope(STREAM_SCOPE).join());
    assertTrue(controller.createStream(STREAM_SCOPE, STREAM_NAME, config).join());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    log.info("Invoking Writer test with Controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build());
    for (int i = 0; i < NUM_EVENTS; i++) {
        String event = "Publish " + i + "\n";
        log.debug("Producing event: {} ", event);
        // any exceptions while writing the event will fail the test.
        writer.writeEvent("", event);
        writer.flush();
    }
    log.info("Invoking Reader test.");
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(STREAM_SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), READER_GROUP, new JavaSerializer<>(), ReaderConfig.builder().build());
    int readCount = 0;
    EventRead<String> event = null;
    do {
        event = reader.readNextEvent(10_000);
        log.debug("Read event: {}.", event.getEvent());
        if (event.getEvent() != null) {
            readCount++;
        }
    // try reading until all the written events are read, else the test will timeout.
    } while ((event.getEvent() != null || event.isCheckpoint()) && readCount < NUM_EVENTS);
    assertEquals("Read count should be equal to write count", NUM_EVENTS, readCount);
}

Example 70 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class BatchClientSimpleTest method batchClientSimpleTest.

/**
 * This test verifies the basic functionality of {@link BatchClientFactory}, including stream metadata checks, segment
 * counts, parallel segment reads and reads with offsets using stream cuts.
 */
@Test
@SuppressWarnings("deprecation")
public void batchClientSimpleTest() {
    final int totalEvents = RG_PARALLELISM * 100;
    final int offsetEvents = RG_PARALLELISM * 20;
    final int batchIterations = 4;
    final Stream stream = Stream.of(SCOPE, STREAM);
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    log.info("Invoking batchClientSimpleTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(SCOPE + "/" + STREAM).build());
    ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    log.info("Writing events to stream");
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, RG_PARALLELISM, offsetEvents);
    Futures.allOf(futures).join();
    // Create a stream cut on the specified offset position.
    Checkpoint cp = readerGroup.initiateCheckpoint("batchClientCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    // Instantiate the batch client and assert it provides correct stream info.
    log.debug("Creating batch client.");
    StreamInfo streamInfo = streamManager.getStreamInfo(SCOPE, stream.getStreamName());
    log.debug("Validating stream metadata fields.");
    assertEquals("Expected Stream name: ", STREAM, streamInfo.getStreamName());
    assertEquals("Expected Scope name: ", SCOPE, streamInfo.getScope());
    // Test that we can read events from parallel segments from an offset onwards.
    log.debug("Reading events from stream cut onwards in parallel.");
    List<SegmentRange> ranges = Lists.newArrayList(batchClient.getSegments(stream, streamCut, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", totalEvents - offsetEvents, readFromRanges(ranges, batchClient));
    // Emulate the behavior of Hadoop client: i) Get tail of Stream, ii) Read from current point until tail, iii) repeat.
    log.debug("Reading in batch iterations.");
    StreamCut currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    int readEvents = 0;
    for (int i = 0; i < batchIterations; i++) {
        writeEvents(clientFactory, STREAM, totalEvents);
        // Read all the existing events in parallel segments from the previous tail to the current one.
        ranges = Lists.newArrayList(batchClient.getSegments(stream, currentTailStreamCut, StreamCut.UNBOUNDED).getIterator());
        assertEquals("Expected number of segments: ", RG_PARALLELISM, ranges.size());
        readEvents += readFromRanges(ranges, batchClient);
        log.debug("Events read in parallel so far: {}.", readEvents);
        currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    }
    assertEquals("Expected events read: .", totalEvents * batchIterations, readEvents);
    // Truncate the stream in first place.
    log.debug("Truncating stream at event {}.", offsetEvents);
    assertTrue(controller.truncateStream(SCOPE, STREAM, streamCut).join());
    // Test the batch client when we select to start reading a Stream from a truncation point.
    StreamCut initialPosition = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getHeadStreamCut();
    List<SegmentRange> newRanges = Lists.newArrayList(batchClient.getSegments(stream, initialPosition, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", (totalEvents - offsetEvents) + totalEvents * batchIterations, readFromRanges(newRanges, batchClient));
    log.debug("Events correctly read from Stream: simple batch client test passed.");
}