Examples with ReaderGroupConfig io.pravega.client.stream.ReaderGroupConfig used on opensource projects

Example 1 with ReaderGroupConfig

use of io.pravega.client.stream.ReaderGroupConfig in project pravega by pravega.

the class ReadTest method readThroughStreamClient.

@Test
public void readThroughStreamClient() throws ReinitializationRequiredException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.flush();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build());
    String read = reader.readNextEvent(5000).getEvent();
    assertEquals(testString, read);
}

Example 2 with ReaderGroupConfig

use of io.pravega.client.stream.ReaderGroupConfig in project pravega by pravega.

the class ReaderGroupTest method testEventHandoff.

@Test(timeout = 20000)
public void testEventHandoff() throws Exception {
    String endpoint = "localhost";
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(SCOPE, endpoint, servicePort);
    streamManager.createScope(SCOPE);
    streamManager.createStream(SCOPE, STREAM_NAME, StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.fixed(2)).build());
    @Cleanup MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(-1).stream(Stream.of(SCOPE, STREAM_NAME)).build();
    streamManager.createReaderGroup(READER_GROUP, groupConfig);
    writeEvents(100, clientFactory);
    ReaderThread r1 = new ReaderThread(20, "Reader1", clientFactory);
    ReaderThread r2 = new ReaderThread(80, "Reader2", clientFactory);
    Thread reader1Thread = new Thread(r1);
    Thread reader2Thread = new Thread(r2);
    reader1Thread.start();
    reader2Thread.start();
    reader1Thread.join();
    reader2Thread.join();
    if (r1.exception.get() != null) {
        throw r1.exception.get();
    }
    if (r2.exception.get() != null) {
        throw r2.exception.get();
    }
    streamManager.deleteReaderGroup(READER_GROUP);
}

Example 3 with ReaderGroupConfig

use of io.pravega.client.stream.ReaderGroupConfig in project pravega by pravega.

the class AutoCheckpointTest method testOnlyOneOutstanding.

@Test(timeout = 30000)
public void testOnlyOneOutstanding() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(1000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader1.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertEquals("As there is a second reader that does not pass the checkpoint, only one should occur", 1, checkpointCount);
}

Example 4 with ReaderGroupConfig

use of io.pravega.client.stream.ReaderGroupConfig in project pravega by pravega.

the class CheckpointTest method testCheckpointAndRestore.

@Test(timeout = 20000)
public void testCheckpointAndRestore() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroupName = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
    ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.flush();
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    EventRead<String> read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    read = reader.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(cpResult).build());
    try {
        reader.readNextEvent(60000);
        fail();
    } catch (ReinitializationRequiredException e) {
    // Expected
    }
    reader.close();
    reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
}

Example 5 with ReaderGroupConfig

use of io.pravega.client.stream.ReaderGroupConfig in project pravega by pravega.

the class AutoCheckpointTest method testCheckpointsOccur.

@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(10000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertTrue("Count was " + checkpointCount, checkpointCount > 5);
    assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}