Examples with StreamConfiguration io.pravega.client.stream.StreamConfiguration used on opensource projects

Example 31 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ControllerServiceTest method sealAStream.

private static void sealAStream(ControllerWrapper controllerWrapper, Controller controller, final ScalingPolicy scalingPolicy, final String scopeSeal, final String streamNameSeal) throws InterruptedException, ExecutionException {
    controllerWrapper.getControllerService().createScope("scopeSeal").get();
    final StreamConfiguration configSeal = StreamConfiguration.builder().scope(scopeSeal).streamName(streamNameSeal).scalingPolicy(scalingPolicy).build();
    assertTrue(controller.createStream(configSeal).get());
    @SuppressWarnings("unused") StreamSegments result = controller.getCurrentSegments(scopeSeal, streamNameSeal).get();
    assertTrue(controller.sealStream(scopeSeal, streamNameSeal).get());
    StreamSegments currentSegs = controller.getCurrentSegments(scopeSeal, streamNameSeal).get();
    assertTrue("FAILURE: No active segments should be present in a sealed stream", currentSegs.getSegments().isEmpty());
}

Example 32 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EventProcessorTest method testEventProcessor.

@Test(timeout = 60000)
public void testEventProcessor() throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    int controllerPort = TestUtils.getAvailableListenPort();
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), true, controllerPort, "localhost", servicePort, 4);
    controllerWrapper.awaitRunning();
    Controller controller = controllerWrapper.getController();
    // Create controller object for testing against a separate controller process.
    // ControllerImpl controller = new ControllerImpl("localhost", 9090);
    final String host = "host";
    final String scope = "controllerScope";
    final String streamName = "stream1";
    final String readerGroup = "readerGroup";
    final CompletableFuture<Boolean> createScopeStatus = controller.createScope(scope);
    if (!createScopeStatus.join()) {
        throw new RuntimeException("Scope already existed");
    }
    final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
    System.err.println(String.format("Creating stream (%s, %s)", scope, streamName));
    CompletableFuture<Boolean> createStatus = controller.createStream(config);
    if (!createStatus.get()) {
        System.err.println("Stream alrady existed, exiting");
        return;
    }
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup EventStreamWriter<TestEvent> producer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
    int expectedSum = input.length * (input.length + 1) / 2;
    for (int i = 0; i < input.length; i++) {
        producer.writeEvent("key", new TestEvent(input[i]));
    }
    producer.writeEvent("key", new TestEvent(-1));
    producer.flush();
    EventProcessorSystem system = new EventProcessorSystemImpl("Controller", host, scope, new ClientFactoryImpl(scope, controller, connectionFactory), new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory));
    CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
    EventProcessorGroupConfig eventProcessorGroupConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroup).streamName(streamName).checkpointConfig(checkpointConfig).build();
    CompletableFuture<Long> result = new CompletableFuture<>();
    // Test case 1. Actor does not throw any exception during normal operation.
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false, result)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(eventProcessorGroupConfig).build();
    @Cleanup EventProcessorGroup<TestEvent> eventProcessorGroup = system.createEventProcessorGroup(eventProcessorConfig, CheckpointStoreFactory.createInMemoryStore());
    Long value = result.join();
    Assert.assertEquals(expectedSum, value.longValue());
    log.info("SUCCESS: received expected sum = " + expectedSum);
}

Example 33 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndChannelLeakTest method testDetectChannelLeakSegmentSealed.

@Test(timeout = 30000)
public void testDetectChannelLeakSegmentSealed() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    // Create a writer.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(SCOPE, new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", READER_GROUP, new JavaSerializer<>(), ReaderConfig.builder().build());
    // Write an event.
    writer.writeEvent("0", "zero").get();
    // Read an event.
    EventRead<String> event = reader1.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("zero", event.getEvent());
    // scale
    Stream stream = new StreamImpl(SCOPE, SCOPE);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    // Write more events.
    writer.writeEvent("0", "one").get();
    writer.writeEvent("0", "two").get();
    writer.writeEvent("1", "three").get();
    // store the open channel count before reading.
    int channelCount = connectionFactory.getActiveChannelCount();
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // Number of sockets will increase by 2 ( +3 for the new segments -1 since the older segment is sealed).
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // no changes to socket count.
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    event = reader1.readNextEvent(10000);
    assertNotNull(event.getEvent());
    // no changes to socket count.
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
}

Example 34 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndChannelLeakTest method testDetectChannelLeakMultiReader.

@Test(timeout = 30000)
public void testDetectChannelLeakMultiReader() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM_NAME).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    // Create a writer.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM_NAME)).build());
    // create a reader.
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Write an event.
    writer.writeEvent("0", "zero").get();
    // Total 4 sockets are open at this point : Writer has 1 connection to segment 0 of stream +
    // Reader has 3 connections (1 metadata client + 1 Writer to _RGreader/0 + 1 Reader to _RGreader/0)
    assertEquals(4, connectionFactory.getActiveChannelCount());
    int channelCount = 4;
    // Read an event.
    EventRead<String> event = reader1.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("zero", event.getEvent());
    // +1 socket to segment 0 of the stream.
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    // scale
    Stream stream = new StreamImpl(SCOPE, STREAM_NAME);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    // No changes to the channel count.
    assertEquals(channelCount, connectionFactory.getActiveChannelCount());
    // Write more events.
    writer.writeEvent("0", "one").get();
    writer.writeEvent("0", "two").get();
    writer.writeEvent("1", "three").get();
    // 2 new connections(+3 connections to the segments 1,2,3 after scale by the writer,
    // -1 connection to segment 0 which is sealed.)
    assertEquals(channelCount + 2, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 2;
    // Add a new reader
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("readerId2", READER_GROUP, serializer, ReaderConfig.builder().build());
    // Creation of a reader will add 3 more connections details similar to the above comment.
    assertEquals(channelCount + 3, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 3;
    event = reader1.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (-1 since segment 0 of stream is sealed + 2 connections to two segments of stream (there are
    // 2 readers and 3 segments and the reader1 will be assigned 2 segments))
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
    channelCount = channelCount + 1;
    event = reader2.readNextEvent(10000);
    assertNotNull(event);
    // +1 connection (a new connection to the remaining stream segment)
    assertEquals(channelCount + 1, connectionFactory.getActiveChannelCount());
}

Example 35 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndTxnWithScaleTest method testTxnWithScale.

@Test(timeout = 10000)
public void testTxnWithScale() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutScaleGracePeriod(10000).transactionTimeoutTime(10000).build());
    Transaction<String> transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest1");
    transaction.commit();
    // scale
    Stream stream = new StreamImpl("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executorService).getFuture().get();
    assertTrue(result);
    transaction = test.beginTxn();
    transaction.writeEvent("0", "txntest2");
    transaction.commit();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest1", event.getEvent());
    event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("txntest2", event.getEvent());
}