Examples with ReaderGroupManager io.pravega.client.admin.ReaderGroupManager used on opensource projects

Example 41 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class EndToEndTxnWithTest method testGetTxnWithScale.

@Test(timeout = 20000)
public void testGetTxnWithScale() throws Exception {
    String streamName = "testGetTxnWithScale";
    final StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    final Serializer<String> serializer = new UTF8StringSerializer();
    final EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(10000).build();
    final Controller controller = PRAVEGA.getLocalController();
    controller.createScope("test").get();
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> streamWriter = clientFactory.createEventWriter(streamName, serializer, writerConfig);
    streamWriter.writeEvent("key", "e").join();
    @Cleanup TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(streamName, serializer, writerConfig);
    Transaction<String> txn = txnWriter.beginTxn();
    txn.writeEvent("key", "1");
    txn.flush();
    // the txn is not yet committed here.
    UUID txnId = txn.getTxnId();
    // scale up stream
    scaleUpStream(streamName);
    // write event using stream writer
    streamWriter.writeEvent("key", "e").join();
    Transaction<String> txn1 = txnWriter.getTxn(txnId);
    txn1.writeEvent("key", "2");
    txn1.flush();
    // commit the transaction
    txn1.commit();
    assertEventuallyEquals(Transaction.Status.COMMITTED, txn1::checkStatus, 5000);
    String group = "testGetTxnWithScale-group";
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream("test/" + streamName).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new UTF8StringSerializer(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(5000);
    assertEquals("e", event.getEvent());
    assertNull(reader.readNextEvent(100).getEvent());
    groupManager.getReaderGroup(group).initiateCheckpoint("cp1", executorService());
    event = reader.readNextEvent(5000);
    assertEquals("Checkpoint event expected", "cp1", event.getCheckpointName());
    event = reader.readNextEvent(5000);
    assertEquals("second event post scale up", "e", event.getEvent());
    assertNull(reader.readNextEvent(100).getEvent());
    groupManager.getReaderGroup(group).initiateCheckpoint("cp2", executorService());
    event = reader.readNextEvent(5000);
    assertEquals("Checkpoint event expected", "cp2", event.getCheckpointName());
    event = reader.readNextEvent(5000);
    assertEquals("txn events", "1", event.getEvent());
    event = reader.readNextEvent(5000);
    assertEquals("txn events", "2", event.getEvent());
}

Example 42 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class EndToEndTxnWithTest method testTxnWithScale.

@Test(timeout = 10000)
public void testTxnWithScale() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("test").get();
    String streamName = "testTxnWithScale";
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup TransactionalEventStreamWriter<String> test = clientFactory.createTransactionalEventWriter("writer", streamName, new UTF8StringSerializer(), EventWriterConfig.builder().transactionTimeoutTime(10000).build());
    Transaction<String> transaction1 = test.beginTxn();
    transaction1.writeEvent("0", "txntest1");
    transaction1.commit();
    assertEventuallyEquals(Transaction.Status.COMMITTED, () -> transaction1.checkStatus(), 5000);
    // scale
    Stream stream = new StreamImpl("test", streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executorService()).getFuture().get();
    assertTrue(result);
    Transaction<String> transaction2 = test.beginTxn();
    transaction2.writeEvent("0", "txntest2");
    transaction2.commit();
    String group = "testTxnWithScale-group";
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream("test/" + streamName).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new UTF8StringSerializer(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(5000);
    assertNotNull(event.getEvent());
    assertEquals("txntest1", event.getEvent());
    assertNull(reader.readNextEvent(100).getEvent());
    groupManager.getReaderGroup(group).initiateCheckpoint("cp", executorService());
    event = reader.readNextEvent(5000);
    assertEquals("cp", event.getCheckpointName());
    event = reader.readNextEvent(5000);
    assertNotNull(event.getEvent());
    assertEquals("txntest2", event.getEvent());
}

Example 43 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytes.

@Test(timeout = 50000)
public void testUnreadBytes() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    String streamName = "testUnreadBytes";
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("unreadbytes").get();
    controller.createStream("unreadbytes", streamName, config).get();
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    String group = "testUnreadBytes-group";
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/" + streamName).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(group);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executorService());
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
}

Example 44 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytesWithCheckpointsAndStreamCuts.

@Test
public void testUnreadBytesWithCheckpointsAndStreamCuts() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    String streamName = "testUnreadBytesWithCheckpointsAndStreamCuts";
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("unreadbytes").get();
    controller.createStream("unreadbytes", streamName, config).get();
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    String group = "testUnreadBytesWithCheckpointsAndStreamCuts-group";
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/" + streamName).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(group);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executorService());
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    // starting from checkpoint "test", data of size 30 is read
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
    // trigger a stream-cut
    CompletableFuture<Map<Stream, StreamCut>> scResult = readerGroup.generateStreamCuts(executorService());
    EventRead<String> scEvent = reader.readNextEvent(15000);
    reader.readNextEvent(100);
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 30);
    // starting from checkpoint "test", data of size 60 is written => stream-cut does not change last checkpointed position
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 60);
}

Example 45 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class SetupUtils method getIntegerReader.

/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    final String readerGroup = "testReaderGroup" + scope + streamName;
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(readerGroupId, readerGroup, new IntegerSerializer(), ReaderConfig.builder().build());
}