Examples with StreamConfiguration io.pravega.client.stream.StreamConfiguration used on opensource projects

Example 46 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ControllerServiceTest method testControllerService.

@Test(timeout = 40000)
public void testControllerService() throws Exception {
    final String scope1 = "scope1";
    final String scope2 = "scope2";
    controllerWrapper.getControllerService().createScope("scope1").get();
    controllerWrapper.getControllerService().createScope("scope2").get();
    Controller controller = controllerWrapper.getController();
    final String streamName1 = "stream1";
    final String streamName2 = "stream2";
    final ScalingPolicy scalingPolicy = ScalingPolicy.fixed(2);
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(scope2).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    final StreamConfiguration config3 = StreamConfiguration.builder().scope(scope1).streamName(streamName2).scalingPolicy(ScalingPolicy.fixed(3)).build();
    createAStream(controller, config1);
    // Same name in different scope
    createAStream(controller, config2);
    // Different name in same scope
    createAStream(controller, config3);
    final String scopeSeal = "scopeSeal";
    final String streamNameSeal = "streamSeal";
    sealAStream(controllerWrapper, controller, scalingPolicy, scopeSeal, streamNameSeal);
    sealASealedStream(controller, scopeSeal, streamNameSeal);
    sealNonExistantStream(controller, scopeSeal);
    streamDuplicationNotAllowed(controller, config1);
    // update stream config section
    updateStreamName(controller, scope1, scalingPolicy);
    updateScalingPolicy(controller, scope1, streamName1);
    updateTargetRate(controller, scope1, streamName1);
    updateScaleFactor(controller, scope1, streamName1);
    updataMinSegmentes(controller, scope1, streamName1);
    updateConfigOfNonExistantStream(controller);
    // get currently active segments
    getActiveSegments(controller, scope1, streamName1);
    getActiveSegmentsForNonExistentStream(controller);
    // get positions at a given time stamp
    getSegmentsAtTime(controller, scope1, streamName1);
    getSegmentsAtTime(controller, scope1, streamName2);
    getSegmentsForNonExistentStream(controller);
    getSegmentsBeforeCreation(controller, scope1, streamName1);
    getSegmentsAfterCreation(controller, scope1, streamName1);
}

Example 47 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ControllerServiceTest method streamMetadataTest.

@Test(timeout = 40000)
public void streamMetadataTest() throws Exception {
    final String scope = "testScope";
    final String stream = "testStream";
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    // Create test scope. This operation should succeed.
    assertTrue(controller.createScope(scope).join());
    // Delete the test scope. This operation should also succeed.
    assertTrue(controller.deleteScope(scope).join());
    // Try creating a stream. It should fail, since the scope does not exist.
    assertFalse(Futures.await(controller.createStream(streamConfiguration)));
    // Again create the scope.
    assertTrue(controller.createScope(scope).join());
    // Try creating the stream again. It should succeed now, since the scope exists.
    assertTrue(controller.createStream(streamConfiguration).join());
    // Delete test scope. This operation should fail, since it is not empty.
    assertFalse(Futures.await(controller.deleteScope(scope)));
    // Delete a non-existent scope.
    assertFalse(controller.deleteScope("non_existent_scope").get());
    // Create a scope with invalid characters. It should fail.
    assertFalse(Futures.await(controller.createScope("abc/def")));
    // Try creating already existing scope.
    assertFalse(controller.createScope(scope).join());
    // Try creating stream with invalid characters. It should fail.
    assertFalse(Futures.await(controller.createStream(StreamConfiguration.builder().scope(scope).streamName("abc/def").scalingPolicy(ScalingPolicy.fixed(1)).build())));
    // Try creating already existing stream.
    assertFalse(controller.createStream(streamConfiguration).join());
}

Example 48 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamMetadataTest method testMetadataOperations.

@Test(timeout = 60000)
public void testMetadataOperations() throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    int controllerPort = TestUtils.getAvailableListenPort();
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, "localhost", servicePort, 4);
    Controller controller = controllerWrapper.getController();
    final String scope1 = "scope1";
    final String streamName1 = "stream1";
    final String scopeSeal = "scopeSeal";
    final String streamNameSeal = "streamSeal";
    final String scope2 = "scope2";
    final String streamName2 = "stream2";
    controllerWrapper.getControllerService().createScope(scope1).get();
    final ScalingPolicy scalingPolicy = ScalingPolicy.fixed(2);
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    // create stream and seal stream
    // CS1:create a stream :given a streamName, scope and config
    assertTrue(controller.createStream(config1).get());
    // Seal a stream given a streamName and scope.
    controllerWrapper.getControllerService().createScope(scopeSeal).get();
    final StreamConfiguration configSeal = StreamConfiguration.builder().scope(scopeSeal).streamName(streamNameSeal).scalingPolicy(scalingPolicy).build();
    assertTrue(controller.createStream(configSeal).get());
    controller.getCurrentSegments(scopeSeal, streamNameSeal).get();
    assertTrue(controller.sealStream(scopeSeal, streamNameSeal).get());
    assertTrue("FAILURE: No active segments should be present in a sealed stream", controller.getCurrentSegments(scopeSeal, streamNameSeal).get().getSegments().isEmpty());
    // Seal an already sealed stream.
    assertTrue(controller.sealStream(scopeSeal, streamNameSeal).get());
    assertTrue("FAILURE: No active segments should be present in a sealed stream", controller.getCurrentSegments(scopeSeal, streamNameSeal).get().getSegments().isEmpty());
    assertThrows("FAILURE: Seal operation on a non-existent stream returned ", controller.sealStream(scopeSeal, "nonExistentStream"), t -> true);
    // CS2:stream duplication not allowed
    assertFalse(controller.createStream(config1).get());
    // CS3:create a stream with same stream name in different scopes
    controllerWrapper.getControllerService().createScope(scope2).get();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(scope2).streamName(streamName1).scalingPolicy(scalingPolicy).build();
    assertTrue(controller.createStream(config2).get());
    // CS4:create a stream with different stream name and config in same scope
    final StreamConfiguration config3 = StreamConfiguration.builder().scope(scope1).streamName(streamName2).scalingPolicy(ScalingPolicy.fixed(3)).build();
    assertTrue(controller.createStream(config3).get());
    // update stream config(update Stream)
    // AS3:update the type of scaling policy
    final StreamConfiguration config6 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(100, 2, 2)).build();
    assertTrue(controller.updateStream(config6).get());
    // AS4:update the target rate of scaling policy
    final StreamConfiguration config7 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 2, 2)).build();
    assertTrue(controller.updateStream(config7).get());
    // AS5:update the scale factor of scaling policy
    final StreamConfiguration config8 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 4, 2)).build();
    assertTrue(controller.updateStream(config8).get());
    // AS6:update the minNumsegments of scaling policy
    final StreamConfiguration config9 = StreamConfiguration.builder().scope(scope1).streamName(streamName1).scalingPolicy(ScalingPolicy.byDataRate(200, 4, 3)).build();
    assertTrue(controller.updateStream(config9).get());
    // AS7:Update configuration of non-existent stream.
    final StreamConfiguration config = StreamConfiguration.builder().scope("scope").streamName("streamName").scalingPolicy(ScalingPolicy.fixed(2)).build();
    CompletableFuture<Boolean> updateStatus = controller.updateStream(config);
    assertThrows("FAILURE: Updating the configuration of a non-existent stream", updateStatus, t -> true);
    // get currently active segments
    // GCS1:get active segments of the stream
    assertFalse(controller.getCurrentSegments(scope1, streamName1).get().getSegments().isEmpty());
    // GCS2:Get active segments for a non-existent stream.
    assertThrows("Active segments cannot be fetched for non existent stream", controller.getCurrentSegments("scope", "streamName"), t -> true);
    // get positions at a given time stamp
    // PS1:get positions at a given time stamp:given stream, time stamp, count
    Stream stream1 = new StreamImpl(scope1, streamName1);
    CompletableFuture<Map<Segment, Long>> segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis());
    assertEquals(2, segments.get().size());
    // PS2:get positions of a stream with different count
    Stream stream2 = new StreamImpl(scope1, streamName2);
    segments = controller.getSegmentsAtTime(stream2, System.currentTimeMillis());
    assertEquals(3, segments.get().size());
    // PS4:get positions at a given timestamp for non-existent stream.
    Stream stream = new StreamImpl("scope", "streamName");
    assertThrows("Fetching segments at given time stamp for non existent stream ", controller.getSegmentsAtTime(stream, System.currentTimeMillis()), t -> true);
    // PS5:Get position at time before stream creation
    segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis() - 36000);
    assertEquals(controller.getCurrentSegments(scope1, streamName1).get().getSegments().size(), segments.get().size());
    // PS6:Get positions at a time in future after stream creation
    segments = controller.getSegmentsAtTime(stream1, System.currentTimeMillis() + 3600);
    assertTrue(!segments.get().isEmpty());
}

Example 49 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndStatsTest method testStatsCount.

@Test(timeout = 10000)
public void testStatsCount() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller);
    @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutScaleGracePeriod(10000).transactionTimeoutTime(10000).build());
    for (int i = 0; i < 10; i++) {
        test.writeEvent("test").get();
    }
    assertEquals(statsRecorder.getSegments().get("test/test/0").get(), 10);
    Transaction<String> transaction = test.beginTxn();
    for (int i = 0; i < 10; i++) {
        transaction.writeEvent("0", "txntest1");
    }
    assertEquals(statsRecorder.getSegments().get("test/test/0").get(), 10);
    transaction.commit();
    Stream stream = new StreamImpl("test", "test");
    while (!controller.checkTransactionStatus(stream, transaction.getTxnId()).get().equals(Transaction.Status.COMMITTED)) {
        Thread.sleep(100);
    }
    assertEquals(statsRecorder.getSegments().get("test/test/0").get(), 20);
}

Example 50 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndTruncationTest method testTruncation.

@Test(timeout = 30000)
public void testTruncation() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 2)).build();
    LocalController controller = (LocalController) controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "truncationTest1").get();
    // scale
    Stream stream = new StreamImpl("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Lists.newArrayList(0, 1), map, executor).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "truncationTest2").get();
    Map<Integer, Long> streamCutPositions = new HashMap<>();
    streamCutPositions.put(2, 0L);
    streamCutPositions.put(3, 0L);
    streamCutPositions.put(4, 0L);
    controller.truncateStream(stream.getStreamName(), stream.getStreamName(), streamCutPositions).join();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
    groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("truncationTest2", event.getEvent());
    event = reader.readNextEvent(1000);
    assertNull(event.getEvent());
// TODO: test more scenarios like: issue #2011
// 1. get a valid stream cut with offset > 0
// validate truncation within a segment
// 2. have an existing reader reading from non truncated segment and then truncate the segment.
// verify that reader gets appropriate response and handles it successfully.
}