Examples with Controller io.pravega.client.stream.impl.Controller used on opensource projects

Example 6 with Controller

use of io.pravega.client.stream.impl.Controller in project pravega by pravega.

the class EventProcessorTest method testEventProcessor.

@Test(timeout = 60000)
public void testEventProcessor() throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    int controllerPort = TestUtils.getAvailableListenPort();
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), true, controllerPort, "localhost", servicePort, 4);
    controllerWrapper.awaitRunning();
    Controller controller = controllerWrapper.getController();
    // Create controller object for testing against a separate controller process.
    // ControllerImpl controller = new ControllerImpl("localhost", 9090);
    final String host = "host";
    final String scope = "controllerScope";
    final String streamName = "stream1";
    final String readerGroup = "readerGroup";
    final CompletableFuture<Boolean> createScopeStatus = controller.createScope(scope);
    if (!createScopeStatus.join()) {
        throw new RuntimeException("Scope already existed");
    }
    final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
    System.err.println(String.format("Creating stream (%s, %s)", scope, streamName));
    CompletableFuture<Boolean> createStatus = controller.createStream(config);
    if (!createStatus.get()) {
        System.err.println("Stream alrady existed, exiting");
        return;
    }
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup EventStreamWriter<TestEvent> producer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
    int expectedSum = input.length * (input.length + 1) / 2;
    for (int i = 0; i < input.length; i++) {
        producer.writeEvent("key", new TestEvent(input[i]));
    }
    producer.writeEvent("key", new TestEvent(-1));
    producer.flush();
    EventProcessorSystem system = new EventProcessorSystemImpl("Controller", host, scope, new ClientFactoryImpl(scope, controller, connectionFactory), new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory));
    CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
    EventProcessorGroupConfig eventProcessorGroupConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroup).streamName(streamName).checkpointConfig(checkpointConfig).build();
    CompletableFuture<Long> result = new CompletableFuture<>();
    // Test case 1. Actor does not throw any exception during normal operation.
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false, result)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(eventProcessorGroupConfig).build();
    @Cleanup EventProcessorGroup<TestEvent> eventProcessorGroup = system.createEventProcessorGroup(eventProcessorConfig, CheckpointStoreFactory.createInMemoryStore());
    Long value = result.join();
    Assert.assertEquals(expectedSum, value.longValue());
    log.info("SUCCESS: received expected sum = " + expectedSum);
}

Example 7 with Controller

use of io.pravega.client.stream.impl.Controller in project pravega by pravega.

the class EndToEndChannelLeakTest method testDetectChannelLeakSegmentSealed.

@Test(timeout = 30000)
public void testDetectChannelLeakSegmentSealed() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    // Create a writer.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(SCOPE, new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", READER_GROUP, new JavaSerializer<>(), ReaderConfig.builder().build());
    // Write an event.
    writer.writeEvent("0", "zero").get();
    // Read an event.
    EventRead<String> event = reader1.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("zero", event.getEvent());
    // scale
    Stream stream = new StreamImpl(SCOPE, SCOPE);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    // Write more events.
    writer.writeEvent("0", "one").get();
    writer.writeEvent("0", "two").get();
    writer.writeEvent("1", "three").get();
    // store the open channel count before reading.
    int channelCount = connectionFactory.getActiveChannelCount();
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // Number of sockets will increase by 2 ( +3 for the new segments -1 since the older segment is sealed).
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // no changes to socket count.
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // no changes to socket count.
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
}

Example 8 with Controller

use of io.pravega.client.stream.impl.Controller in project pravega by pravega.

the class EndToEndChannelLeakTest method testDetectChannelLeakMultiReader.

@Test(timeout = 30000)
public void testDetectChannelLeakMultiReader() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    // Create a writer.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM_NAME)).build());
    // create a reader.
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Write an event.
    writer.writeEvent("0", "zero").get();
    // Total 4 sockets are open at this point : Writer has 1 connection to segment 0 of stream +
    // Reader has 3 connections (1 metadata client + 1 Writer to _RGreader/0 + 1 Reader to _RGreader/0)
    assertEquals(4, connectionFactory.getActiveChannelCount());
    int channelCount = 4;
    // Read an event.
    EventRead<String> event = reader1.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("zero", event.getEvent());
    // +1 socket to segment 0 of the stream.
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    // scale
    Stream stream = new StreamImpl(SCOPE, STREAM_NAME);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    // No changes to the channel count.
    assertEquals(channelCount, connectionFactory.getActiveChannelCount());
    // Write more events.
    writer.writeEvent("0", "one").get();
    writer.writeEvent("0", "two").get();
    writer.writeEvent("1", "three").get();
    // 2 new connections(+3 connections to the segments 1,2,3 after scale by the writer,
    // -1 connection to segment 0 which is sealed.)
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 2;
    // Add a new reader
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("readerId2", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Creation of a reader will add 3 more connections details similar to the above comment.
    assertEquals(channelCount + 3, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 3;
    event = reader1.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (-1 since segment 0 of stream is sealed + 2 connections to two segments of stream (there are
    // 2 readers and 3 segments and the reader1 will be assigned 2 segments))
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    event = reader2.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (a new connection to the remaining stream segment)
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
}

Example 9 with Controller

use of io.pravega.client.stream.impl.Controller in project pravega by pravega.

the class EndToEndTxnWithScaleTest method testTxnWithScale.

@Test(timeout = 10000)
public void testTxnWithScale() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutScaleGracePeriod(10000).transactionTimeoutTime(10000).build());
    Transaction<String> transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest1");
    transaction.commit();
    // scale
    Stream stream = new StreamImpl("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executorService).getFuture().get();
    assertTrue(result);
    transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest2");
    transaction.commit();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest1", event.getEvent());
    event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest2", event.getEvent());
}

Example 10 with Controller

use of io.pravega.client.stream.impl.Controller in project pravega by pravega.

the class ScaleTest method main.

public static void main(String[] args) throws Exception {
    try {
        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        int port = Config.SERVICE_PORT;
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
        server.startListening();
        // Create controller object for testing against a separate controller report.
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
        Controller controller = controllerWrapper.getController();
        final String scope = "scope";
        controllerWrapper.getControllerService().createScope(scope).get();
        final String streamName = "stream1";
        final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
        Stream stream = new StreamImpl(scope, streamName);
        log.info("Creating stream {}/{}", scope, streamName);
        if (!controller.createStream(config).get()) {
            log.error("Stream already existed, exiting");
            return;
        }
        // Test 1: scale stream: split one segment into two
        log.info("Scaling stream {}/{}, splitting one segment into two", scope, streamName);
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 0.5);
        map.put(0.5, 1.0);
        if (!controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get()) {
            log.error("Scale stream: splitting segment into two failed, exiting");
            return;
        }
        // Test 2: scale stream: merge two segments into one
        log.info("Scaling stream {}/{}, merging two segments into one", scope, streamName);
        CompletableFuture<Boolean> scaleResponseFuture = controller.scaleStream(stream, Arrays.asList(1, 2), Collections.singletonMap(0.0, 1.0), executor).getFuture();
        if (!scaleResponseFuture.get()) {
            log.error("Scale stream: merging two segments into one failed, exiting");
            return;
        }
        // Test 3: create a transaction, and try scale operation, it should fail with precondition check failure
        CompletableFuture<TxnSegments> txnFuture = controller.createTransaction(stream, 5000, 29000);
        TxnSegments transaction = txnFuture.get();
        if (transaction == null) {
            log.error("Create transaction failed, exiting");
            return;
        }
        log.info("Scaling stream {}/{}, splitting one segment into two, while transaction is ongoing", scope, streamName);
        scaleResponseFuture = controller.scaleStream(stream, Collections.singletonList(3), map, executor).getFuture();
        CompletableFuture<Boolean> future = scaleResponseFuture.whenComplete((r, e) -> {
            if (e != null) {
                log.error("Failed: scale with ongoing transaction.", e);
            } else if (getAndHandleExceptions(controller.checkTransactionStatus(stream, transaction.getTxnId()), RuntimeException::new) != Transaction.Status.OPEN) {
                log.info("Success: scale with ongoing transaction.");
            } else {
                log.error("Failed: scale with ongoing transaction.");
            }
        });
        CompletableFuture<Void> statusFuture = controller.abortTransaction(stream, transaction.getTxnId());
        statusFuture.get();
        future.get();
        log.info("All scaling test PASSED");
        executor.shutdown();
        System.exit(0);
    } catch (Throwable t) {
        log.error("test failed with {}", t);
        System.exit(-1);
    }
}