Examples with Controller io.pravega.client.control.impl.Controller used on opensource projects

Example 51 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytes.

@Test(timeout = 50000)
public void testUnreadBytes() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    String streamName = "testUnreadBytes";
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("unreadbytes").get();
    controller.createStream("unreadbytes", streamName, config).get();
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    String group = "testUnreadBytes-group";
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/" + streamName).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(group);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executorService());
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
}

Example 52 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytesWithCheckpointsAndStreamCuts.

@Test
public void testUnreadBytesWithCheckpointsAndStreamCuts() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    String streamName = "testUnreadBytesWithCheckpointsAndStreamCuts";
    Controller controller = PRAVEGA.getLocalController();
    controller.createScope("unreadbytes").get();
    controller.createStream("unreadbytes", streamName, config).get();
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    String group = "testUnreadBytesWithCheckpointsAndStreamCuts-group";
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/" + streamName).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(group);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executorService());
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    // starting from checkpoint "test", data of size 30 is read
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
    // trigger a stream-cut
    CompletableFuture<Map<Stream, StreamCut>> scResult = readerGroup.generateStreamCuts(executorService());
    EventRead<String> scEvent = reader.readNextEvent(15000);
    reader.readNextEvent(100);
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 30);
    // starting from checkpoint "test", data of size 60 is written => stream-cut does not change last checkpointed position
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 60);
}

Example 53 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class DebugStreamSegmentsTest method createStream.

private void createStream(String streamName) throws Exception {
    Controller controller = controllerWrapper.getController();
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 4)).retentionPolicy(RetentionPolicy.bySizeBytes(100 * 1024)).build();
    controller.createStream(SCOPE, streamName, config).get();
}

Example 54 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class ReaderGroupNotificationTest method testEndOfStreamNotifications.

@Test(timeout = 40000)
public void testEndOfStreamNotifications() throws Exception {
    final String streamName = "stream2";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE, 0L).get();
    controller.createStream(SCOPE, streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    // seal stream
    assertTrue(controller.sealStream(SCOPE, streamName).get());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).groupRefreshTimeMillis(0).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("reader");
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
    // Add segment event listener
    Listener<EndOfDataNotification> l1 = notification -> {
        listenerInvoked.set(true);
        listenerLatch.release();
    };
    EndOfDataNotifier endOfDataNotifier = (EndOfDataNotifier) readerGroup.getEndOfDataNotifier(executorService());
    endOfDataNotifier.registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(10000);
    assertEquals("data1", event1.getEvent());
    EventRead<String> emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp", executorService());
    EventRead<String> cpEvent = reader1.readNextEvent(10000);
    assertTrue(cpEvent.isCheckpoint());
    EventRead<String> event2 = reader1.readNextEvent(10000);
    assertEquals("data2", event2.getEvent());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp2", executorService());
    cpEvent = reader1.readNextEvent(10000);
    assertTrue(cpEvent.isCheckpoint());
    emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    endOfDataNotifier.pollNow();
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
}

Example 55 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class ReaderGroupNotificationTest method testSegmentNotifications.

@Test(timeout = 40000)
public void testSegmentNotifications() throws Exception {
    final String streamName = "stream1";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE, 0L).get();
    controller.createStream(SCOPE, streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).groupRefreshTimeMillis(0).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("reader");
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
    val notificationResults = new ArrayBlockingQueue<SegmentNotification>(2);
    // Add segment event listener
    Listener<SegmentNotification> l1 = notification -> {
        log.info("Number of Segments: {}, Number of Readers: {}", notification.getNumOfSegments(), notification.getNumOfReaders());
        notificationResults.add(notification);
    };
    SegmentNotifier segmentNotifier = (SegmentNotifier) readerGroup.getSegmentNotifier(executorService());
    segmentNotifier.registerListener(l1);
    // Read first event and validate notification.
    EventRead<String> event1 = reader1.readNextEvent(5000);
    assertEquals("data1", event1.getEvent());
    segmentNotifier.pollNow();
    SegmentNotification initialSegmentNotification = notificationResults.take();
    assertNotNull(initialSegmentNotification);
    assertEquals(1, initialSegmentNotification.getNumOfReaders());
    assertEquals(1, initialSegmentNotification.getNumOfSegments());
    EventRead<String> emptyEvent = reader1.readNextEvent(0);
    assertNull(emptyEvent.getEvent());
    assertFalse(emptyEvent.isCheckpoint());
    readerGroup.initiateCheckpoint("cp", executorService());
    EventRead<String> cpEvent = reader1.readNextEvent(1000);
    assertTrue(cpEvent.isCheckpoint());
    // Read second event and validate notification.
    EventRead<String> event2 = reader1.readNextEvent(10000);
    assertEquals("data2", event2.getEvent());
    segmentNotifier.pollNow();
    SegmentNotification segmentNotificationPostScale = notificationResults.take();
    assertEquals(1, segmentNotificationPostScale.getNumOfReaders());
    assertEquals(2, segmentNotificationPostScale.getNumOfSegments());
}