Examples with ScalingPolicy io.pravega.client.stream.ScalingPolicy used on opensource projects

Example 21 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ReaderGroupNotificationTest method testEndOfStreamNotifications.

@Test(timeout = 40000)
public void testEndOfStreamNotifications() throws Exception {
    final String streamName = "stream2";
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    // seal stream
    assertTrue(controller.sealStream(SCOPE, streamName).get());
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    // Add segment event listener
    Listener<EndOfDataNotification> l1 = notification -> {
        listenerInvoked.set(true);
        listenerLatch.release();
    };
    readerGroup.getEndOfDataNotifier(executor).registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(10000);
    EventRead<String> event2 = reader1.readNextEvent(10000);
    EventRead<String> event3 = reader1.readNextEvent(10000);
    assertNotNull(event1);
    assertEquals("data1", event1.getEvent());
    assertNotNull(event2);
    assertEquals("data2", event2.getEvent());
    assertNull(event3.getEvent());
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
}

Example 22 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ReaderGroupNotificationTest method testSegmentNotifications.

@Test(timeout = 40000)
public void testSegmentNotifications() throws Exception {
    final String streamName = "stream1";
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    // scale
    Stream stream = new StreamImpl(SCOPE, streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "data2").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    // Add segment event listener
    Listener<SegmentNotification> l1 = notification -> {
        listenerInvoked.set(true);
        numberOfReaders.set(notification.getNumOfReaders());
        numberOfSegments.set(notification.getNumOfSegments());
        listenerLatch.release();
    };
    readerGroup.getSegmentNotifier(executor).registerListener(l1);
    EventRead<String> event1 = reader1.readNextEvent(15000);
    EventRead<String> event2 = reader1.readNextEvent(15000);
    assertNotNull(event1);
    assertEquals("data1", event1.getEvent());
    assertNotNull(event2);
    assertEquals("data2", event2.getEvent());
    listenerLatch.await();
    assertTrue("Listener invoked", listenerInvoked.get());
    assertEquals(2, numberOfSegments.get());
    assertEquals(1, numberOfReaders.get());
}

Example 23 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class EndToEndTransactionTest method main.

@Test
public static void main(String[] args) throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int port = Config.SERVICE_PORT;
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    Thread.sleep(1000);
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
    Controller controller = controllerWrapper.getController();
    controllerWrapper.awaitRunning();
    final String testScope = "testScope";
    final String testStream = "testStream";
    if (!controller.createScope(testScope).get()) {
        log.error("FAILURE: Error creating test scope");
        return;
    }
    ScalingPolicy policy = ScalingPolicy.fixed(5);
    StreamConfiguration streamConfig = StreamConfiguration.builder().scope(testScope).streamName(testStream).scalingPolicy(policy).build();
    if (!controller.createStream(streamConfig).get()) {
        log.error("FAILURE: Error creating test stream");
        return;
    }
    final long txnTimeout = 4000;
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(testScope, controller);
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(testStream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(txnTimeout).transactionTimeoutScaleGracePeriod(txnTimeout).build());
    // region Successful commit tests
    Transaction<String> transaction = producer.beginTxn();
    for (int i = 0; i < 1; i++) {
        String event = "\n Transactional Publish \n";
        log.info("Producing event: " + event);
        transaction.writeEvent("", event);
        transaction.flush();
        Thread.sleep(500);
    }
    CompletableFuture<Object> commit = CompletableFuture.supplyAsync(() -> {
        try {
            transaction.commit();
        } catch (Exception e) {
            log.warn("Error committing transaction", e);
        }
        return null;
    });
    commit.join();
    Transaction.Status txnStatus = transaction.checkStatus();
    assertTrue(txnStatus == Transaction.Status.COMMITTING || txnStatus == Transaction.Status.COMMITTED);
    log.info("SUCCESS: successful in committing transaction. Transaction status=" + txnStatus);
    Thread.sleep(2000);
    txnStatus = transaction.checkStatus();
    assertTrue(txnStatus == Transaction.Status.COMMITTED);
    log.info("SUCCESS: successfully committed transaction. Transaction status=" + txnStatus);
    // endregion
    // region Successful abort tests
    Transaction<String> transaction2 = producer.beginTxn();
    for (int i = 0; i < 1; i++) {
        String event = "\n Transactional Publish \n";
        log.info("Producing event: " + event);
        transaction2.writeEvent("", event);
        transaction2.flush();
        Thread.sleep(500);
    }
    CompletableFuture<Object> drop = CompletableFuture.supplyAsync(() -> {
        try {
            transaction2.abort();
        } catch (Exception e) {
            log.warn("Error aborting transaction", e);
        }
        return null;
    });
    drop.join();
    Transaction.Status txn2Status = transaction2.checkStatus();
    assertTrue(txn2Status == Transaction.Status.ABORTING || txn2Status == Transaction.Status.ABORTED);
    log.info("SUCCESS: successful in dropping transaction. Transaction status=" + txn2Status);
    Thread.sleep(2000);
    txn2Status = transaction2.checkStatus();
    assertTrue(txn2Status == Transaction.Status.ABORTED);
    log.info("SUCCESS: successfully aborted transaction. Transaction status=" + txn2Status);
    // endregion
    // region Successful timeout tests
    Transaction<String> tx1 = producer.beginTxn();
    Thread.sleep((long) (1.3 * txnTimeout));
    Transaction.Status txStatus = tx1.checkStatus();
    Assert.assertTrue(Transaction.Status.ABORTING == txStatus || Transaction.Status.ABORTED == txStatus);
    log.info("SUCCESS: successfully aborted transaction after timeout. Transaction status=" + txStatus);
    // endregion
    // region Ping failure due to controller going into disconnection state
    // Fill in these tests once we have controller.stop() implemented.
    System.exit(0);
}

Example 24 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ControllerServiceTest method testControllerService.

@Test(timeout = 40000)
public void testControllerService() throws Exception {
    final String scope1 = "scope1";
    final String scope2 = "scope2";
    controllerWrapper.getControllerService().createScope("scope1").get();
    controllerWrapper.getControllerService().createScope("scope2").get();
    Controller controller = controllerWrapper.getController();
    final String streamName1 = "stream1";
    final String streamName2 = "stream2";
    final ScalingPolicy scalingPolicy = ScalingPolicy.fixed(2);
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(scope2).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    final StreamConfiguration config3 = StreamConfiguration.builder().scope(scope1).streamName(streamName2).scalingPolicy(ScalingPolicy.fixed(3)).build();
    createAStream(controller, config1);
    // Same name in different scope
    createAStream(controller, config2);
    // Different name in same scope
    createAStream(controller, config3);
    final String scopeSeal = "scopeSeal";
    final String streamNameSeal = "streamSeal";
    sealAStream(controllerWrapper, controller, scalingPolicy, scopeSeal, streamNameSeal);
    sealASealedStream(controller, scopeSeal, streamNameSeal);
    sealNonExistantStream(controller, scopeSeal);
    streamDuplicationNotAllowed(controller, config1);
    // update stream config section
    updateStreamName(controller, scope1, scalingPolicy);
    updateScalingPolicy(controller, scope1, streamName1);
    updateTargetRate(controller, scope1, streamName1);
    updateScaleFactor(controller, scope1, streamName1);
    updataMinSegmentes(controller, scope1, streamName1);
    updateConfigOfNonExistantStream(controller);
    // get currently active segments
    getActiveSegments(controller, scope1, streamName1);
    getActiveSegmentsForNonExistentStream(controller);
    // get positions at a given time stamp
    getSegmentsAtTime(controller, scope1, streamName1);
    getSegmentsAtTime(controller, scope1, streamName2);
    getSegmentsForNonExistentStream(controller);
    getSegmentsBeforeCreation(controller, scope1, streamName1);
    getSegmentsAfterCreation(controller, scope1, streamName1);
}

Example 25 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataTest method testMetadataOperations.

@Test(timeout = 60000)
public void testMetadataOperations() throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    int controllerPort = TestUtils.getAvailableListenPort();
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, "localhost", servicePort, 4);
    Controller controller = controllerWrapper.getController();
    final String scope1 = "scope1";
    final String streamName1 = "stream1";
    final String scopeSeal = "scopeSeal";
    final String streamNameSeal = "streamSeal";
    final String scope2 = "scope2";
    final String streamName2 = "stream2";
    controllerWrapper.getControllerService().createScope(scope1).get();
    final ScalingPolicy scalingPolicy = ScalingPolicy.fixed(2);
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    // create stream and seal stream
    // CS1:create a stream :given a streamName, scope and config
    assertTrue(controller.createStream(config1).get());
    // Seal a stream given a streamName and scope.
    controllerWrapper.getControllerService().createScope(scopeSeal).get();
    final StreamConfiguration configSeal = StreamConfiguration.builder().scope(scopeSeal).streamName(streamNameSeal).scalingPolicy(scalingPolicy).build();
    assertTrue(controller.createStream(configSeal).get());
    controller.getCurrentSegments(scopeSeal, streamNameSeal).get();
    assertTrue(controller.sealStream(scopeSeal, streamNameSeal).get());
    assertTrue("FAILURE: No active segments should be present in a sealed stream", controller.getCurrentSegments(scopeSeal, streamNameSeal).get().getSegments().isEmpty());
    // Seal an already sealed stream.
    assertTrue(controller.sealStream(scopeSeal, streamNameSeal).get());
    assertTrue("FAILURE: No active segments should be present in a sealed stream", controller.getCurrentSegments(scopeSeal, streamNameSeal).get().getSegments().isEmpty());
    assertThrows("FAILURE: Seal operation on a non-existent stream returned ", controller.sealStream(scopeSeal, "nonExistentStream"), t -> true);
    // CS2:stream duplication not allowed
    assertFalse(controller.createStream(config1).get());
    // CS3:create a stream with same stream name in different scopes
    controllerWrapper.getControllerService().createScope(scope2).get();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(scope2).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    assertTrue(controller.createStream(config2).get());
    // CS4:create a stream with different stream name and config in same scope
    final StreamConfiguration config3 = StreamConfiguration.builder().scope(scope1).streamName(streamName2).scalingPolicy(ScalingPolicy.fixed(3)).build();
    assertTrue(controller.createStream(config3).get());
    // update stream config(update Stream)
    // AS3:update the type of scaling policy
    final StreamConfiguration config6 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(100, 2, 2)).build();
    assertTrue(controller.updateStream(config6).get());
    // AS4:update the target rate of scaling policy
    final StreamConfiguration config7 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 2, 2)).build();
    assertTrue(controller.updateStream(config7).get());
    // AS5:update the scale factor of scaling policy
    final StreamConfiguration config8 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 4, 2)).build();
    assertTrue(controller.updateStream(config8).get());
    // AS6:update the minNumsegments of scaling policy
    final StreamConfiguration config9 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 4, 3)).build();
    assertTrue(controller.updateStream(config9).get());
    // AS7:Update configuration of non-existent stream.
    final StreamConfiguration config = StreamConfiguration.builder().scope("scope").streamName("streamName").scalingPolicy(ScalingPolicy.fixed(2)).build();
    CompletableFuture<Boolean> updateStatus = controller.updateStream(config);
    assertThrows("FAILURE: Updating the configuration of a non-existent stream", updateStatus, t -> true);
    // get currently active segments
    // GCS1:get active segments of the stream
    assertFalse(controller.getCurrentSegments(scope1, streamName1).get().getSegments().isEmpty());
    // GCS2:Get active segments for a non-existent stream.
    assertThrows("Active segments cannot be fetched for non existent stream", controller.getCurrentSegments("scope", "streamName"), t -> true);
    // get positions at a given time stamp
    // PS1:get positions at a given time stamp:given stream, time stamp, count
    Stream stream1 = new StreamImpl(scope1, streamName1);
    CompletableFuture<Map<Segment, Long>> segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis());
    assertEquals(2, segments.get().size());
    // PS2:get positions of a stream with different count
    Stream stream2 = new StreamImpl(scope1, streamName2);
    segments = controller.getSegmentsAtTime(stream2, System.currentTimeMillis());
    assertEquals(3, segments.get().size());
    // PS4:get positions at a given timestamp for non-existent stream.
    Stream stream = new StreamImpl("scope", "streamName");
    assertThrows("Fetching segments at given time stamp for non existent stream ", controller.getSegmentsAtTime(stream, System.currentTimeMillis()), t -> true);
    // PS5:Get position at time before stream creation
    segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis() - 36000);
    assertEquals(controller.getCurrentSegments(scope1, streamName1).get().getSegments().size(), segments.get().size());
    // PS6:Get positions at a time in future after stream creation
    segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis() + 3600);
    assertTrue(!segments.get().isEmpty());
}