Examples with ReaderGroupManager io.pravega.client.admin.ReaderGroupManager used on opensource projects

Example 11 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class MultiReadersEndToEndTest method runTest.

private void runTest(final Set<String> streamNames, final int numParallelReaders, final int numSegments) throws Exception {
    @Cleanup StreamManager streamManager = StreamManager.create(ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    streamManager.createScope(SETUP_UTILS.getScope());
    streamNames.stream().forEach(stream -> {
        streamManager.createStream(SETUP_UTILS.getScope(), stream, StreamConfiguration.builder().scope(SETUP_UTILS.getScope()).streamName(stream).scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
        log.info("Created stream: {}", stream);
    });
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope(SETUP_UTILS.getScope(), ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    streamNames.stream().forEach(stream -> {
        EventStreamWriter<Integer> eventWriter = clientFactory.createEventWriter(stream, new IntegerSerializer(), EventWriterConfig.builder().build());
        for (Integer i = 0; i < NUM_TEST_EVENTS; i++) {
            eventWriter.writeEvent(String.valueOf(i), i);
        }
        eventWriter.flush();
        log.info("Wrote {} events", NUM_TEST_EVENTS);
    });
    final String readerGroupName = "testreadergroup" + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SETUP_UTILS.getScope(), ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    ReaderGroupConfig.ReaderGroupConfigBuilder builder = ReaderGroupConfig.builder();
    streamNames.forEach(s -> builder.stream(Stream.of(SETUP_UTILS.getScope(), s)));
    readerGroupManager.createReaderGroup(readerGroupName, builder.build());
    Collection<Integer> read = readAllEvents(numParallelReaders, clientFactory, readerGroupName, numSegments);
    Assert.assertEquals(NUM_TEST_EVENTS * streamNames.size(), read.size());
    // Check unique events.
    Assert.assertEquals(NUM_TEST_EVENTS, new TreeSet<>(read).size());
    readerGroupManager.deleteReaderGroup(readerGroupName);
}

Example 12 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class MultiSegmentStoreTest method testReadWrite.

private void testReadWrite() {
    List<URI> ctlURIs = this.controllerInstance.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    String scope = "testscope" + RandomStringUtils.randomAlphanumeric(10);
    String stream = "teststream" + RandomStringUtils.randomAlphanumeric(10);
    @Cleanup StreamManager streamManager = StreamManager.create(ClientConfig.builder().controllerURI(controllerUri).build());
    Assert.assertTrue(streamManager.createScope(scope));
    // Create stream with large number of segments so that most segment containers are used.
    Assert.assertTrue(streamManager.createStream(scope, stream, StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(ScalingPolicy.fixed(10)).build()));
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerUri).build());
    log.info("Invoking writer with controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(stream, new JavaSerializer<>(), EventWriterConfig.builder().build());
    final int numEvents = 1000;
    final String fixedEvent = "testevent";
    for (int i = 0; i < numEvents; i++) {
        log.debug("Producing event: {} ", fixedEvent);
        writer.writeEvent(String.valueOf(i), fixedEvent);
    }
    writer.flush();
    log.info("Invoking reader with controller URI: {}", controllerUri);
    final String readerGroup = "testreadergroup" + RandomStringUtils.randomAlphanumeric(10);
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, ClientConfig.builder().controllerURI(controllerUri).build());
    groupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, stream)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
    for (int i = 0; i < numEvents; i++) {
        try {
            String event = reader.readNextEvent(60000).getEvent();
            Assert.assertEquals(fixedEvent, event);
        } catch (ReinitializationRequiredException e) {
            log.error("Unexpected request to reinitialize {}", e);
            throw new IllegalStateException("Unexpected request to reinitialize");
        }
    }
}

Example 13 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class SetupUtils method getIntegerReader.

/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    final String readerGroup = "testReaderGroup" + scope + streamName;
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build());
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(readerGroupId, readerGroup, new IntegerSerializer(), ReaderConfig.builder().build());
}

Example 14 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class ControllerRestApiTest method restApiTests.

@Test
public void restApiTests() {
    Invocation.Builder builder;
    Response response;
    restServerURI = SETUP_UTILS.getControllerRestUri().toString();
    log.info("REST Server URI: {}", restServerURI);
    // TEST REST server status, ping test
    resourceURl = new StringBuilder(restServerURI).append("/ping").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("Ping test", OK.getStatusCode(), response.getStatus());
    log.info("REST Server is running. Ping successful.");
    final String scope1 = RandomStringUtils.randomAlphanumeric(10);
    final String stream1 = RandomStringUtils.randomAlphanumeric(10);
    // TEST CreateScope POST http://controllerURI:Port/v1/scopes
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
    webTarget = client.target(resourceURl);
    final CreateScopeRequest createScopeRequest = new CreateScopeRequest();
    createScopeRequest.setScopeName(scope1);
    builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
    response = builder.post(Entity.json(createScopeRequest));
    assertEquals("Create scope status", CREATED.getStatusCode(), response.getStatus());
    Assert.assertEquals("Create scope response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
    log.info("Create scope: {} successful ", scope1);
    // TEST CreateStream POST  http://controllerURI:Port/v1/scopes/{scopeName}/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
    webTarget = client.target(resourceURl);
    CreateStreamRequest createStreamRequest = new CreateStreamRequest();
    ScalingConfig scalingConfig = new ScalingConfig();
    scalingConfig.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
    scalingConfig.setTargetRate(2);
    scalingConfig.scaleFactor(2);
    scalingConfig.minSegments(2);
    RetentionConfig retentionConfig = new RetentionConfig();
    retentionConfig.setType(RetentionConfig.TypeEnum.LIMITED_DAYS);
    retentionConfig.setValue(123L);
    createStreamRequest.setStreamName(stream1);
    createStreamRequest.setScalingPolicy(scalingConfig);
    createStreamRequest.setRetentionPolicy(retentionConfig);
    builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
    response = builder.post(Entity.json(createStreamRequest));
    assertEquals("Create stream status", CREATED.getStatusCode(), response.getStatus());
    final StreamProperty streamPropertyResponse = response.readEntity(StreamProperty.class);
    assertEquals("Scope name in response", scope1, streamPropertyResponse.getScopeName());
    assertEquals("Stream name in response", stream1, streamPropertyResponse.getStreamName());
    log.info("Create stream: {} successful", stream1);
    // Test listScopes  GET http://controllerURI:Port/v1/scopes/{scopeName}/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("List scopes", OK.getStatusCode(), response.getStatus());
    log.info("List scopes successful");
    // Test listStream GET /v1/scopes/scope1/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    Assert.assertEquals("List streams size", 1, response.readEntity(StreamsList.class).getStreams().size());
    log.info("List streams successful");
    // Test getScope
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get scope status", OK.getStatusCode(), response.getStatus());
    assertEquals("Get scope scope1 response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
    log.info("Get scope successful");
    // Test updateStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    UpdateStreamRequest updateStreamRequest = new UpdateStreamRequest();
    ScalingConfig scalingConfig1 = new ScalingConfig();
    scalingConfig1.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
    scalingConfig1.setTargetRate(2);
    // update existing scaleFactor from 2 to 3
    scalingConfig1.scaleFactor(3);
    // update existing minSegments from 2 to 4
    scalingConfig1.minSegments(4);
    updateStreamRequest.setScalingPolicy(scalingConfig1);
    updateStreamRequest.setRetentionPolicy(retentionConfig);
    response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(updateStreamRequest));
    assertEquals("Update stream status", OK.getStatusCode(), response.getStatus());
    assertEquals("Verify updated property", 4, response.readEntity(StreamProperty.class).getScalingPolicy().getMinSegments().intValue());
    log.info("Update stream successful");
    // Test getStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get stream status", OK.getStatusCode(), response.getStatus());
    assertEquals("Get stream stream1 response", stream1, response.readEntity(StreamProperty.class).getStreamName());
    log.info("Get stream successful");
    // Test updateStreamState
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1 + "/state").toString();
    StreamState streamState = new StreamState();
    streamState.setStreamState(StreamState.StreamStateEnum.SEALED);
    response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(streamState));
    assertEquals("UpdateStreamState status", OK.getStatusCode(), response.getStatus());
    assertEquals("UpdateStreamState status in response", streamState.getStreamState(), response.readEntity(StreamState.class).getStreamState());
    log.info("Update stream state successful");
    // Test deleteStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    response = client.target(resourceURl).request().delete();
    assertEquals("DeleteStream status", NO_CONTENT.getStatusCode(), response.getStatus());
    log.info("Delete stream successful");
    // Test deleteScope
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
    response = client.target(resourceURl).request().delete();
    assertEquals("Get scope status", NO_CONTENT.getStatusCode(), response.getStatus());
    log.info("Delete Scope successful");
    // Test reader groups APIs.
    // Prepare the streams and readers using the admin client.
    final String testScope = RandomStringUtils.randomAlphanumeric(10);
    final String testStream1 = RandomStringUtils.randomAlphanumeric(10);
    final String testStream2 = RandomStringUtils.randomAlphanumeric(10);
    URI controllerUri = SETUP_UTILS.getControllerUri();
    @Cleanup("shutdown") InlineExecutor inlineExecutor = new InlineExecutor();
    try (StreamManager streamManager = new StreamManagerImpl(createController(controllerUri, inlineExecutor))) {
        log.info("Creating scope: {}", testScope);
        streamManager.createScope(testScope);
        log.info("Creating stream: {}", testStream1);
        StreamConfiguration streamConf1 = StreamConfiguration.builder().scope(testScope).streamName(testStream1).scalingPolicy(ScalingPolicy.fixed(1)).build();
        streamManager.createStream(testScope, testStream1, streamConf1);
        log.info("Creating stream: {}", testStream2);
        StreamConfiguration streamConf2 = StreamConfiguration.builder().scope(testScope).streamName(testStream2).scalingPolicy(ScalingPolicy.fixed(1)).build();
        streamManager.createStream(testScope, testStream2, streamConf2);
    }
    final String readerGroupName1 = RandomStringUtils.randomAlphanumeric(10);
    final String readerGroupName2 = RandomStringUtils.randomAlphanumeric(10);
    final String reader1 = RandomStringUtils.randomAlphanumeric(10);
    final String reader2 = RandomStringUtils.randomAlphanumeric(10);
    try (ClientFactory clientFactory = new ClientFactoryImpl(testScope, createController(controllerUri, inlineExecutor));
        ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(testScope, ClientConfig.builder().controllerURI(controllerUri).build())) {
        readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
        readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
        clientFactory.createReader(reader1, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
        clientFactory.createReader(reader2, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
    }
    // Test fetching readergroups.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups").toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroups status", OK.getStatusCode(), response.getStatus());
    ReaderGroupsList readerGroupsList = response.readEntity(ReaderGroupsList.class);
    assertEquals("Get readergroups size", 2, readerGroupsList.getReaderGroups().size());
    assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName1)));
    assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName2)));
    log.info("Get readergroups successful");
    // Test fetching readergroup info.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + readerGroupName1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", OK.getStatusCode(), response.getStatus());
    ReaderGroupProperty readerGroupProperty = response.readEntity(ReaderGroupProperty.class);
    assertEquals("Get readergroup name", readerGroupName1, readerGroupProperty.getReaderGroupName());
    assertEquals("Get readergroup scope name", testScope, readerGroupProperty.getScopeName());
    assertEquals("Get readergroup streams size", 2, readerGroupProperty.getStreamList().size());
    assertTrue(readerGroupProperty.getStreamList().contains(testStream1));
    assertTrue(readerGroupProperty.getStreamList().contains(testStream2));
    assertEquals("Get readergroup onlinereaders size", 2, readerGroupProperty.getOnlineReaderIds().size());
    assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader1));
    assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader2));
    // Test readergroup or scope not found.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + "unknownreadergroup").toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + "unknownscope" + "/readergroups/" + readerGroupName1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
    log.info("Get readergroup properties successful");
    log.info("Test restApiTests passed successfully!");
}

Example 15 with ReaderGroupManager

use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.

the class StreamCutsTest method testReaderGroupCuts.

@Test(timeout = 40000)
public void testReaderGroupCuts() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("test").streamName("test").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream(config).get();
    @Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "fpj was here").get();
    writer.writeEvent("0", "fpj was here again").get();
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
    ReaderGroup readerGroup = groupManager.createReaderGroup("cuts", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "cuts", new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("fpj was here", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("fpj was here again", secondEvent.getEvent());
    Map<Stream, StreamCut> cuts = readerGroup.getStreamCuts();
    validateCuts(readerGroup, cuts, Collections.singleton("test/test/0"));
    // Scale the stream to verify that we get more segments in the cut.
    Stream stream = Stream.of("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 1st scaling");
    writer.writeEvent("0", "fpj was here again").get();
    writer.writeEvent("1", "fpj was here again").get();
    reader.readNextEvent(15000);
    cuts = readerGroup.getStreamCuts();
    HashSet<String> segmentNames = new HashSet<>();
    segmentNames.add("test/test/1");
    segmentNames.add("test/test/2");
    validateCuts(readerGroup, cuts, Collections.unmodifiableSet(segmentNames));
    // Scale down to verify that the number drops back.
    map = new HashMap<>();
    map.put(0.0, 1.0);
    ArrayList<Integer> toSeal = new ArrayList<>();
    toSeal.add(1);
    toSeal.add(2);
    result = controller.scaleStream(stream, Collections.unmodifiableList(toSeal), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 2nd scaling");
    writer.writeEvent("0", "fpj was here again").get();
    reader.readNextEvent(15000);
    reader.readNextEvent(15000);
    cuts = readerGroup.getStreamCuts();
    validateCuts(readerGroup, cuts, Collections.singleton("test/test/3"));
    // Scale up to 4 segments again.
    map = new HashMap<>();
    map.put(0.0, 0.25);
    map.put(0.25, 0.5);
    map.put(0.5, 0.75);
    map.put(0.75, 1.0);
    result = controller.scaleStream(stream, Collections.singletonList(3), map, executor).getFuture().get();
    assertTrue(result);
    log.info("Finished 3rd scaling");
    writer.writeEvent("0", "fpj was here again").get();
    reader.readNextEvent(15000);
    cuts = readerGroup.getStreamCuts();
    segmentNames = new HashSet<>();
    segmentNames.add("test/test/4");
    segmentNames.add("test/test/5");
    segmentNames.add("test/test/6");
    segmentNames.add("test/test/7");
    validateCuts(readerGroup, cuts, Collections.unmodifiableSet(segmentNames));
}