Examples with Segment io.pravega.controller.store.stream.Segment used on opensource projects

Example 6 with Segment

use of io.pravega.controller.store.stream.Segment in project pravega by pravega.

the class StreamMetadataTasksTest method manualScaleTest.

@Test(timeout = 30000)
public void manualScaleTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(1);
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName("test").scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    ScaleResponse scaleOpResult = streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(0), newRanges, 30, null).get();
    assertEquals(ScaleStreamStatus.STARTED, scaleOpResult.getStatus());
    OperationContext context = streamStorePartialMock.createContext(SCOPE, "test");
    assertEquals(streamStorePartialMock.getState(SCOPE, "test", false, context, executor).get(), State.ACTIVE);
    // Now when startScale runs even after that we should get the state as active.
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, "test", Collections.singletonList(0), newRanges, 30, true, null, executor).get();
    assertEquals(response.getActiveEpoch(), 0);
    assertEquals(streamStorePartialMock.getState(SCOPE, "test", true, context, executor).get(), State.ACTIVE);
    AssertExtensions.assertThrows("", () -> streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, "test", Collections.singletonList(0), response.getSegmentsCreated(), response.getActiveEpoch(), 30, context, executor).get(), ex -> Exceptions.unwrap(ex) instanceof StoreException.IllegalStateException);
    List<Segment> segments = streamMetadataTasks.startScale((ScaleOpEvent) requestEventWriter.getEventQueue().take(), true, context, "").get();
    assertTrue(segments.stream().anyMatch(x -> x.getNumber() == 1 && x.getKeyStart() == 0.0 && x.getKeyEnd() == 0.5));
    assertTrue(segments.stream().anyMatch(x -> x.getNumber() == 2 && x.getKeyStart() == 0.5 && x.getKeyEnd() == 1.0));
}

Example 7 with Segment

use of io.pravega.controller.store.stream.Segment in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore(1, executor);
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    doReturn(CompletableFuture.completedFuture(false)).when(streamStorePartialMock).isTransactionOngoing(anyString(), anyString(), any(), // mock only isTransactionOngoing call.
    any());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key"));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, hostStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new SealStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), executor);
    consumer = new ControllerService(streamStorePartialMock, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStorePartialMock.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
}

Example 8 with Segment

use of io.pravega.controller.store.stream.Segment in project pravega by pravega.

the class StreamMetadataTasksTest method scale.

private void scale(String scope, String stream, Map<Integer, Long> sealedSegmentsWithSize, List<AbstractMap.SimpleEntry<Double, Double>> newSegments) {
    long scaleTs = System.currentTimeMillis();
    ArrayList<Integer> sealedSegments = Lists.newArrayList(sealedSegmentsWithSize.keySet());
    StartScaleResponse response = streamStorePartialMock.startScale(scope, stream, sealedSegments, newSegments, scaleTs, false, null, executor).join();
    final List<Segment> scale1SegmentsCreated = response.getSegmentsCreated();
    streamStorePartialMock.setState(scope, stream, State.SCALING, null, executor).join();
    streamStorePartialMock.scaleNewSegmentsCreated(scope, stream, sealedSegments, scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    streamStorePartialMock.scaleSegmentsSealed(scope, stream, sealedSegmentsWithSize, scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    streamStorePartialMock.setState(scope, stream, State.ACTIVE, null, executor).join();
}

Example 9 with Segment

use of io.pravega.controller.store.stream.Segment in project pravega by pravega.

the class StreamMetaDataTests method testGetScalingEvents.

/**
 * Test for getScalingEvents REST API.
 */
@Test
public void testGetScalingEvents() throws Exception {
    String resourceURI = getURI() + "v1/scopes/scope1/streams/stream1/scaling-events";
    List<ScaleMetadata> scaleMetadataList = new ArrayList<>();
    /* Test to get scaling events

        There are 5 scale events in final list.
        Filter 'from' and 'to' is also tested here.
        Event1 is before 'from'
        Event2 is before 'from'
        Event3 and Event4 are between 'from' and 'to'
        Event5 is after 'to'
        Response contains 3 events : Event2 acts as reference event. Event3 and Event4 fall between 'from' and 'to'.
         */
    Segment segment1 = new Segment(0, System.currentTimeMillis(), 0.00, 0.50);
    Segment segment2 = new Segment(1, System.currentTimeMillis(), 0.50, 1.00);
    List<Segment> segmentList1 = Arrays.asList(segment1, segment2);
    ScaleMetadata scaleMetadata1 = new ScaleMetadata(System.currentTimeMillis() / 2, segmentList1);
    Segment segment3 = new Segment(2, System.currentTimeMillis(), 0.00, 0.40);
    Segment segment4 = new Segment(3, System.currentTimeMillis(), 0.40, 1.00);
    List<Segment> segmentList2 = Arrays.asList(segment3, segment4);
    ScaleMetadata scaleMetadata2 = new ScaleMetadata(1 + System.currentTimeMillis() / 2, segmentList2);
    long fromDateTime = System.currentTimeMillis();
    Segment segment5 = new Segment(4, System.currentTimeMillis(), 0.00, 0.50);
    Segment segment6 = new Segment(5, System.currentTimeMillis(), 0.50, 1.00);
    List<Segment> segmentList3 = Arrays.asList(segment5, segment6);
    ScaleMetadata scaleMetadata3 = new ScaleMetadata(System.currentTimeMillis(), segmentList3);
    Segment segment7 = new Segment(6, System.currentTimeMillis(), 0.00, 0.25);
    Segment segment8 = new Segment(7, System.currentTimeMillis(), 0.25, 1.00);
    List<Segment> segmentList4 = Arrays.asList(segment7, segment8);
    ScaleMetadata scaleMetadata4 = new ScaleMetadata(System.currentTimeMillis(), segmentList4);
    long toDateTime = System.currentTimeMillis();
    Segment segment9 = new Segment(8, System.currentTimeMillis(), 0.00, 0.40);
    Segment segment10 = new Segment(9, System.currentTimeMillis(), 0.40, 1.00);
    List<Segment> segmentList5 = Arrays.asList(segment9, segment10);
    ScaleMetadata scaleMetadata5 = new ScaleMetadata(toDateTime * 2, segmentList5);
    // HistoryRecords.readAllRecords returns a list of records in decreasing order
    // so we add the elements in reverse order as well to simulate that behavior
    scaleMetadataList.add(scaleMetadata5);
    scaleMetadataList.add(scaleMetadata4);
    scaleMetadataList.add(scaleMetadata3);
    scaleMetadataList.add(scaleMetadata2);
    scaleMetadataList.add(scaleMetadata1);
    when(mockControllerService.getScaleRecords(scope1, stream1)).thenReturn(CompletableFuture.completedFuture(scaleMetadataList));
    Response response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 200, response.getStatus());
    assertTrue(response.bufferEntity());
    List<ScaleMetadata> scaleMetadataListResponse = response.readEntity(new GenericType<List<ScaleMetadata>>() {
    });
    assertEquals("List Size", 3, scaleMetadataListResponse.size());
    scaleMetadataListResponse.forEach(data -> {
        log.warn("Here");
        data.getSegments().forEach(segment -> {
            log.debug("Checking segment number: " + segment.getNumber());
            assertTrue("Event 1 shouldn't be included", segment.getNumber() != 0);
        });
    });
    // Test for large number of scaling events.
    scaleMetadataList.clear();
    scaleMetadataList.addAll(Collections.nCopies(50000, scaleMetadata3));
    when(mockControllerService.getScaleRecords(scope1, stream1)).thenReturn(CompletableFuture.completedFuture(scaleMetadataList));
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 200, response.getStatus());
    assertTrue(response.bufferEntity());
    scaleMetadataListResponse = response.readEntity(new GenericType<List<ScaleMetadata>>() {
    });
    assertEquals("List Size", 50000, scaleMetadataListResponse.size());
    // Test for getScalingEvents for invalid scope/stream.
    final CompletableFuture<List<ScaleMetadata>> completableFuture1 = new CompletableFuture<>();
    completableFuture1.completeExceptionally(StoreException.create(StoreException.Type.DATA_NOT_FOUND, "stream1"));
    when(mockControllerService.getScaleRecords("scope1", "stream1")).thenReturn(completableFuture1);
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime).queryParam("to", toDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 404, response.getStatus());
    // Test for getScalingEvents for bad request.
    // from > to is tested here
    when(mockControllerService.getScaleRecords("scope1", "stream1")).thenReturn(CompletableFuture.completedFuture(scaleMetadataList));
    response = addAuthHeaders(client.target(resourceURI).queryParam("from", fromDateTime * 2).queryParam("to", fromDateTime).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 400, response.getStatus());
    // Test for getScalingEvents failure.
    final CompletableFuture<List<ScaleMetadata>> completableFuture = new CompletableFuture<>();
    completableFuture.completeExceptionally(new Exception());
    when(mockControllerService.getScaleRecords("scope1", "stream1")).thenReturn(completableFuture);
    response = addAuthHeaders(client.target(resourceURI).request()).buildGet().invoke();
    assertEquals("Get Scaling Events response code", 500, response.getStatus());
}

Example 10 with Segment

use of io.pravega.controller.store.stream.Segment in project pravega by pravega.

the class ControllerServiceTest method setup.

@Before
public void setup() throws ExecutionException, InterruptedException {
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final ScalingPolicy policy2 = ScalingPolicy.fixed(3);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration configuration2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy2).build();
    // createScope
    streamStore.createScope(SCOPE).get();
    // region createStream
    startTs = System.currentTimeMillis();
    OperationContext context = streamStore.createContext(SCOPE, stream1);
    streamStore.createStream(SCOPE, stream1, configuration1, startTs, context, executor).get();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, context, executor);
    OperationContext context2 = streamStore.createContext(SCOPE, stream2);
    streamStore.createStream(SCOPE, stream2, configuration2, startTs, context2, executor).get();
    streamStore.setState(SCOPE, stream2, State.ACTIVE, context2, executor);
    // endregion
    // region scaleSegments
    SimpleEntry<Double, Double> segment1 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    scaleTs = System.currentTimeMillis();
    StartScaleResponse startScaleResponse = streamStore.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), startTs + 20, false, null, executor).get();
    List<Segment> segmentCreated = startScaleResponse.getSegmentsCreated();
    streamStore.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.0, 0.5);
    SimpleEntry<Double, Double> segment4 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment5 = new SimpleEntry<>(0.75, 1.0);
    sealedSegments = Arrays.asList(0, 1, 2);
    startScaleResponse = streamStore.startScale(SCOPE, stream2, sealedSegments, Arrays.asList(segment3, segment4, segment5), startTs + 20, false, null, executor).get();
    segmentCreated = startScaleResponse.getSegmentsCreated();
    streamStore.setState(SCOPE, stream2, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream2, sealedSegments, segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream2, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentCreated, startScaleResponse.getActiveEpoch(), scaleTs, null, executor).get();
// endregion
}