Examples with ScaleMetadata io.pravega.controller.store.stream.ScaleMetadata used on opensource projects

Example 1 with ScaleMetadata

use of io.pravega.controller.store.stream.ScaleMetadata in project pravega by pravega.

the class ControllerRestApiTest method restApiTests.

@Test
public void restApiTests() {
    Invocation.Builder builder;
    Response response;
    restServerURI = SETUP_UTILS.getControllerRestUri().toString();
    log.info("REST Server URI: {}", restServerURI);
    // TEST REST server status, ping test
    resourceURl = new StringBuilder(restServerURI).append("/ping").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("Ping test", OK.getStatusCode(), response.getStatus());
    log.info("REST Server is running. Ping successful.");
    final String scope1 = RandomStringUtils.randomAlphanumeric(10);
    final String stream1 = RandomStringUtils.randomAlphanumeric(10);
    // TEST CreateScope POST http://controllerURI:Port/v1/scopes
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
    webTarget = client.target(resourceURl);
    final CreateScopeRequest createScopeRequest = new CreateScopeRequest();
    createScopeRequest.setScopeName(scope1);
    builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
    response = builder.post(Entity.json(createScopeRequest));
    assertEquals("Create scope status", CREATED.getStatusCode(), response.getStatus());
    Assert.assertEquals("Create scope response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
    log.info("Create scope: {} successful ", scope1);
    // Create another scope for empty stream test later.
    final String scope2 = RandomStringUtils.randomAlphanumeric(10);
    final CreateScopeRequest createScopeRequest1 = new CreateScopeRequest();
    createScopeRequest1.setScopeName(scope2);
    builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
    response = builder.post(Entity.json(createScopeRequest1));
    assertEquals("Create scope status", CREATED.getStatusCode(), response.getStatus());
    Assert.assertEquals("Create scope response", scope2, response.readEntity(ScopeProperty.class).getScopeName());
    // TEST CreateStream POST  http://controllerURI:Port/v1/scopes/{scopeName}/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
    webTarget = client.target(resourceURl);
    CreateStreamRequest createStreamRequest = new CreateStreamRequest();
    ScalingConfig scalingConfig = new ScalingConfig();
    scalingConfig.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
    scalingConfig.setTargetRate(2);
    scalingConfig.scaleFactor(2);
    scalingConfig.minSegments(2);
    RetentionConfig retentionConfig = new RetentionConfig();
    retentionConfig.setType(RetentionConfig.TypeEnum.LIMITED_DAYS);
    retentionConfig.setValue(123L);
    TagsList tagsList = new TagsList();
    tagsList.add("testTag");
    createStreamRequest.setStreamName(stream1);
    createStreamRequest.setScalingPolicy(scalingConfig);
    createStreamRequest.setRetentionPolicy(retentionConfig);
    createStreamRequest.setStreamTags(tagsList);
    createStreamRequest.setTimestampAggregationTimeout(1000L);
    createStreamRequest.setRolloverSizeBytes(1024L);
    builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
    response = builder.post(Entity.json(createStreamRequest));
    assertEquals("Create stream status", CREATED.getStatusCode(), response.getStatus());
    final StreamProperty streamPropertyResponse = response.readEntity(StreamProperty.class);
    assertEquals("Scope name in response", scope1, streamPropertyResponse.getScopeName());
    assertEquals("Stream name in response", stream1, streamPropertyResponse.getStreamName());
    assertEquals("TimestampAggregationTimeout in response", 1000L, (long) streamPropertyResponse.getTimestampAggregationTimeout());
    assertEquals("RolloverSizeBytes in response", 1024L, (long) streamPropertyResponse.getRolloverSizeBytes());
    log.info("Create stream: {} successful", stream1);
    // Test listScopes  GET http://controllerURI:Port/v1/scopes/{scopeName}/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("List scopes", OK.getStatusCode(), response.getStatus());
    log.info("List scopes successful");
    // Test listStream GET /v1/scopes/scope1/streams
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
    webTarget = client.target(resourceURl);
    builder = webTarget.request();
    response = builder.get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    Assert.assertEquals("List streams size", 1, response.readEntity(StreamsList.class).getStreams().size());
    log.info("List streams successful");
    // Test listStream GET /v1/scopes/scope1/streams for tags
    response = client.target(resourceURl).queryParam("filter_type", "tag").queryParam("filter_value", "testTag").request().get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    Assert.assertEquals("List streams size", 1, response.readEntity(StreamsList.class).getStreams().size());
    response = client.target(resourceURl).queryParam("filter_type", "tag").queryParam("filter_value", "randomTag").request().get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    Assert.assertEquals("List streams size", 0, response.readEntity(StreamsList.class).getStreams().size());
    log.info("List streams with tag successful");
    response = client.target(resourceURl).queryParam("filter_type", "showInternalStreams").request().get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    assertTrue(response.readEntity(StreamsList.class).getStreams().get(0).getStreamName().startsWith("_MARK"));
    log.info("List streams with showInternalStreams successful");
    // Test for the case when the scope is empty.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope2 + "/streams").toString();
    response = client.target(resourceURl).request().get();
    assertEquals("List streams", OK.getStatusCode(), response.getStatus());
    Assert.assertEquals("List streams size", 0, response.readEntity(StreamsList.class).getStreams().size());
    // Test getScope
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get scope status", OK.getStatusCode(), response.getStatus());
    assertEquals("Get scope scope1 response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
    log.info("Get scope successful");
    // Test updateStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    UpdateStreamRequest updateStreamRequest = new UpdateStreamRequest();
    ScalingConfig scalingConfig1 = new ScalingConfig();
    scalingConfig1.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
    scalingConfig1.setTargetRate(2);
    // update existing scaleFactor from 2 to 3
    scalingConfig1.scaleFactor(3);
    // update existing minSegments from 2 to 4
    scalingConfig1.minSegments(4);
    updateStreamRequest.setScalingPolicy(scalingConfig1);
    updateStreamRequest.setRetentionPolicy(retentionConfig);
    updateStreamRequest.setTimestampAggregationTimeout(2000L);
    updateStreamRequest.setRolloverSizeBytes(2048L);
    response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(updateStreamRequest));
    assertEquals("Update stream status", OK.getStatusCode(), response.getStatus());
    assertEquals("Verify updated property", 4, response.readEntity(StreamProperty.class).getScalingPolicy().getMinSegments().intValue());
    log.info("Update stream successful");
    // Test scaling event list GET /v1/scopes/scope1/streams/stream1
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1 + "/scaling-events").toString();
    response = client.target(resourceURl).queryParam("from", 0L).queryParam("to", System.currentTimeMillis()).request().get();
    List<ScaleMetadata> scaleMetadataListResponse = response.readEntity(new GenericType<List<ScaleMetadata>>() {
    });
    assertEquals(2, scaleMetadataListResponse.size());
    assertEquals(2, scaleMetadataListResponse.get(0).getSegments().size());
    assertEquals(4, scaleMetadataListResponse.get(1).getSegments().size());
    // Test getStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get stream status", OK.getStatusCode(), response.getStatus());
    StreamProperty responseProperty = response.readEntity(StreamProperty.class);
    assertEquals("Get stream stream1 response", stream1, responseProperty.getStreamName());
    assertEquals("Get stream stream1 response TimestampAggregationTimeout", (long) responseProperty.getTimestampAggregationTimeout(), 2000L);
    assertEquals("Get stream stream1 RolloverSizeBytes", (long) responseProperty.getRolloverSizeBytes(), 2048L);
    log.info("Get stream successful");
    // Test updateStreamState
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1 + "/state").toString();
    StreamState streamState = new StreamState();
    streamState.setStreamState(StreamState.StreamStateEnum.SEALED);
    response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(streamState));
    assertEquals("UpdateStreamState status", OK.getStatusCode(), response.getStatus());
    assertEquals("UpdateStreamState status in response", streamState.getStreamState(), response.readEntity(StreamState.class).getStreamState());
    log.info("Update stream state successful");
    // Test deleteStream
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
    response = client.target(resourceURl).request().delete();
    assertEquals("DeleteStream status", NO_CONTENT.getStatusCode(), response.getStatus());
    log.info("Delete stream successful");
    // Test deleteScope
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
    response = client.target(resourceURl).request().delete();
    assertEquals("Get scope status", NO_CONTENT.getStatusCode(), response.getStatus());
    log.info("Delete Scope successful");
    // Test reader groups APIs.
    // Prepare the streams and readers using the admin client.
    final String testScope = RandomStringUtils.randomAlphanumeric(10);
    final String testStream1 = RandomStringUtils.randomAlphanumeric(10);
    final String testStream2 = RandomStringUtils.randomAlphanumeric(10);
    URI controllerUri = SETUP_UTILS.getControllerUri();
    @Cleanup("shutdown") InlineExecutor inlineExecutor = new InlineExecutor();
    ClientConfig clientConfig = ClientConfig.builder().build();
    try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
        StreamManager streamManager = new StreamManagerImpl(createController(controllerUri, inlineExecutor), cp)) {
        log.info("Creating scope: {}", testScope);
        streamManager.createScope(testScope);
        log.info("Creating stream: {}", testStream1);
        StreamConfiguration streamConf1 = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
        streamManager.createStream(testScope, testStream1, streamConf1);
        log.info("Creating stream: {}", testStream2);
        StreamConfiguration streamConf2 = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
        streamManager.createStream(testScope, testStream2, streamConf2);
    }
    final String readerGroupName1 = RandomStringUtils.randomAlphanumeric(10);
    final String readerGroupName2 = RandomStringUtils.randomAlphanumeric(10);
    final String reader1 = RandomStringUtils.randomAlphanumeric(10);
    final String reader2 = RandomStringUtils.randomAlphanumeric(10);
    try (ClientFactoryImpl clientFactory = new ClientFactoryImpl(testScope, createController(controllerUri, inlineExecutor), clientConfig);
        ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(testScope, ClientConfig.builder().controllerURI(controllerUri).build())) {
        readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
        readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
        clientFactory.createReader(reader1, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
        clientFactory.createReader(reader2, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
    }
    // Test fetching readergroups.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups").toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroups status", OK.getStatusCode(), response.getStatus());
    ReaderGroupsList readerGroupsList = response.readEntity(ReaderGroupsList.class);
    assertEquals("Get readergroups size", 2, readerGroupsList.getReaderGroups().size());
    assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName1)));
    assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName2)));
    log.info("Get readergroups successful");
    // Test fetching readergroup info.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + readerGroupName1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", OK.getStatusCode(), response.getStatus());
    ReaderGroupProperty readerGroupProperty = response.readEntity(ReaderGroupProperty.class);
    assertEquals("Get readergroup name", readerGroupName1, readerGroupProperty.getReaderGroupName());
    assertEquals("Get readergroup scope name", testScope, readerGroupProperty.getScopeName());
    assertEquals("Get readergroup streams size", 2, readerGroupProperty.getStreamList().size());
    assertTrue(readerGroupProperty.getStreamList().contains(Stream.of(testScope, testStream1).getScopedName()));
    assertTrue(readerGroupProperty.getStreamList().contains(Stream.of(testScope, testStream2).getScopedName()));
    assertEquals("Get readergroup onlinereaders size", 2, readerGroupProperty.getOnlineReaderIds().size());
    assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader1));
    assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader2));
    // Test readergroup or scope not found.
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + "unknownreadergroup").toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
    resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + "unknownscope" + "/readergroups/" + readerGroupName1).toString();
    response = client.target(resourceURl).request().get();
    assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
    log.info("Get readergroup properties successful");
    log.info("Test restApiTests passed successfully!");
}

Example 2 with ScaleMetadata

use of io.pravega.controller.store.stream.ScaleMetadata in project pravega by pravega.

the class StreamMetaDataTests method testGetScalingEvents.

/**
 * Test for getScalingEvents REST API.
 */
@Test(timeout = 30000)
public void testGetScalingEvents() throws Exception {
    String resourceURI = getURI() + "v1/scopes/scope1/streams/stream1/scaling-events";
    List<ScaleMetadata> scaleMetadataList = new ArrayList<>();
    /* Test to get scaling events

        There are 5 scale events in final list.
        Filter 'from' and 'to' is also tested here.
        Event1 is before 'from'
        Event2 is before 'from'
        Event3 and Event4 are between 'from' and 'to'
        Event5 is after 'to'
        Response contains 3 events : Event2 acts as reference event. Event3 and Event4 fall between 'from' and 'to'.
         */
    Segment segment1 = new Segment(0, 0, System.currentTimeMillis(), 0.00, 0.50);
    Segment segment2 = new Segment(1, 0, System.currentTimeMillis(), 0.50, 1.00);
    List<Segment> segmentList1 = Arrays.asList(segment1, segment2);
    ScaleMetadata scaleMetadata1 = new ScaleMetadata(System.currentTimeMillis() / 2, segmentList1, 0L, 0L);
    Segment segment3 = new Segment(2, 0, System.currentTimeMillis(), 0.00, 0.40);
    Segment segment4 = new Segment(3, 0, System.currentTimeMillis(), 0.40, 1.00);
    List<Segment> segmentList2 = Arrays.asList(segment3, segment4);
    ScaleMetadata scaleMetadata2 = new ScaleMetadata(1 + System.currentTimeMillis() / 2, segmentList2, 1L, 1L);
    long fromDateTime = System.currentTimeMillis();
    Segment segment5 = new Segment(4, 0, System.currentTimeMillis(), 0.00, 0.50);
    Segment segment6 = new Segment(5, 0, System.currentTimeMillis(), 0.50, 1.00);
    List<Segment> segmentList3 = Arrays.asList(segment5, segment6);
    ScaleMetadata scaleMetadata3 = new ScaleMetadata(System.currentTimeMillis(), segmentList3, 1L, 1L);
    Segment segment7 = new Segment(6, 0, System.currentTimeMillis(), 0.00, 0.25);
    Segment segment8 = new Segment(7, 0, System.currentTimeMillis(), 0.25, 1.00);
    List<Segment> segmentList4 = Arrays.asList(segment7, segment8);
    ScaleMetadata scaleMetadata4 = new ScaleMetadata(System.currentTimeMillis(), segmentList4, 1L, 1L);
    long toDateTime = System.currentTimeMillis();
    Segment segment9 = new Segment(8, 0, System.currentTimeMillis(), 0.00, 0.40);
    Segment segment10 = new Segment(9, 0, System.currentTimeMillis(), 0.40, 1.00);
    List<Segment> segmentList5 = Arrays.asList(segment9, segment10);
    ScaleMetadata scaleMetadata5 = new ScaleMetadata(toDateTime * 2, segmentList5, 1L, 1L);
    // HistoryRecords.readAllRecords returns a list of records in decreasing order
    // so we add the elements in reverse order as well to simulate that behavior
    scaleMetadataList.add(scaleMetadata5);
    scaleMetadataList.add(scaleMetadata4);
    scaleMetadataList.add(scaleMetadata3);
    scaleMetadataList.add(scaleMetadata2);
    scaleMetadataList.add(scaleMetadata1);
    doAnswer(x -> CompletableFuture.completedFuture(scaleMetadataList)).when(mockControllerService).getScaleRecords(anyString(), anyString(), anyLong(), anyLong(), anyLong());
    Response response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 200, response.getStatus());
    assertTrue(response.bufferEntity());
    List<ScaleMetadata> scaleMetadataListResponse = response.readEntity(new GenericType<List<ScaleMetadata>>() {
    });
    assertEquals("List Size", 3, scaleMetadataListResponse.size());
    scaleMetadataListResponse.forEach(data -> {
        log.warn("Here");
        data.getSegments().forEach(segment -> {
            log.debug("Checking segment number: " + segment.segmentId());
            assertTrue("Event 1 shouldn't be included", segment.segmentId() != 0L);
        });
    });
    // Test for large number of scaling events.
    scaleMetadataList.clear();
    scaleMetadataList.addAll(Collections.nCopies(50000, scaleMetadata3));
    doAnswer(x -> CompletableFuture.completedFuture(scaleMetadataList)).when(mockControllerService).getScaleRecords(anyString(), anyString(), anyLong(), anyLong(), anyLong());
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 200, response.getStatus());
    assertTrue(response.bufferEntity());
    scaleMetadataListResponse = response.readEntity(new GenericType<List<ScaleMetadata>>() {
    });
    assertEquals("List Size", 50000, scaleMetadataListResponse.size());
    // Test for getScalingEvents for invalid scope/stream.
    final CompletableFuture<List<ScaleMetadata>> completableFuture1 = new CompletableFuture<>();
    completableFuture1.completeExceptionally(StoreException.create(StoreException.Type.DATA_NOT_FOUND, "stream1"));
    doAnswer(x -> completableFuture1).when(mockControllerService).getScaleRecords(anyString(), anyString(), anyLong(), anyLong(), anyLong());
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 404, response.getStatus());
    // Test for getScalingEvents for bad request.
    // 1. Missing query parameters are validated here.
    response = addAuthHeaders(client.target(resourceURI).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 400, response.getStatus());
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 400, response.getStatus());
    // 2. from > to is tested here.
    doAnswer(x -> CompletableFuture.completedFuture(scaleMetadataList)).when(mockControllerService).getScaleRecords(anyString(), anyString(), anyLong(), anyLong(), anyLong());
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime * 2).queryParam("to", fromDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 400, response.getStatus());
    // Test for getScalingEvents failure.
    final CompletableFuture<List<ScaleMetadata>> completableFuture = new CompletableFuture<>();
    completableFuture.completeExceptionally(new Exception());
    doAnswer(x -> completableFuture).when(mockControllerService).getScaleRecords(anyString(), anyString(), anyLong(), anyLong(), anyLong());
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 500, response.getStatus());
}