Examples with ScalingPolicy io.pravega.client.stream.ScalingPolicy used on opensource projects

Example 11 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class TaskTest method setUp.

@Before
public void setUp() throws ExecutionException, InterruptedException {
    final String stream2 = "stream2";
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final ScalingPolicy policy2 = ScalingPolicy.fixed(3);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration configuration2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy2).build();
    // region createStream
    streamStore.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, stream1, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, null, executor).join();
    streamStore.createStream(SCOPE, stream2, configuration2, start, null, executor).join();
    streamStore.setState(SCOPE, stream2, State.ACTIVE, null, executor).join();
    // endregion
    // region scaleSegments
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStore.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStore.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment3 = new AbstractMap.SimpleEntry<>(0.0, 0.5);
    AbstractMap.SimpleEntry<Double, Double> segment4 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment5 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments1 = Arrays.asList(0, 1, 2);
    response = streamStore.startScale(SCOPE, stream2, sealedSegments1, Arrays.asList(segment3, segment4, segment5), start + 20, false, null, executor).get();
    segmentsCreated = response.getSegmentsCreated();
    streamStore.setState(SCOPE, stream2, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream2, sealedSegments1, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream2, sealedSegments1.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
// endregion
}

Example 12 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ControllerServiceWithZKStreamTest method getSegmentsImmediatelyFollowingTest.

@Test(timeout = 5000)
public void getSegmentsImmediatelyFollowingTest() throws Exception {
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Start time  when stream is created.
    long start = System.currentTimeMillis();
    // Create stream and scope
    Controller.CreateScopeStatus scopeStatus = consumer.createScope(SCOPE).join();
    assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, scopeStatus.getStatus());
    Controller.CreateStreamStatus streamStatus = consumer.createStream(configuration1, start).get();
    assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamStatus.getStatus());
    List<Controller.SegmentRange> currentSegments = consumer.getCurrentSegments(SCOPE, STREAM).get();
    assertEquals(2, currentSegments.size());
    // scale segment 1 which has key range from 0.5 to 1.0 at time: start+20
    Map<Double, Double> keyRanges = new HashMap<>(2);
    keyRanges.put(0.5, 0.75);
    keyRanges.put(0.75, 1.0);
    assertFalse(consumer.checkScale(SCOPE, STREAM, 0).get().getStatus().equals(Controller.ScaleStatusResponse.ScaleStatus.SUCCESS));
    Controller.ScaleResponse scaleStatus = consumer.scale(SCOPE, STREAM, Arrays.asList(1), keyRanges, start + 20).get();
    assertEquals(Controller.ScaleResponse.ScaleStreamStatus.STARTED, scaleStatus.getStatus());
    AtomicBoolean done = new AtomicBoolean(false);
    Futures.loop(() -> !done.get(), () -> consumer.checkScale(SCOPE, STREAM, scaleStatus.getEpoch()).thenAccept(x -> done.set(x.getStatus().equals(Controller.ScaleStatusResponse.ScaleStatus.SUCCESS))), executor).get();
    // After scale the current number of segments is 3;
    List<Controller.SegmentRange> currentSegmentsAfterScale = consumer.getCurrentSegments(SCOPE, STREAM).get();
    assertEquals(3, currentSegmentsAfterScale.size());
    Map<Controller.SegmentRange, List<Integer>> successorsOfSeg1 = consumer.getSegmentsImmediatelyFollowing(ModelHelper.createSegmentId(SCOPE, STREAM, 1)).get();
    // two segments follow segment 1
    assertEquals(2, successorsOfSeg1.size());
    Map<Controller.SegmentRange, List<Integer>> successorsOfSeg0 = consumer.getSegmentsImmediatelyFollowing(ModelHelper.createSegmentId(SCOPE, STREAM, 0)).get();
    // no segments follow segment 0
    assertEquals(0, successorsOfSeg0.size());
}

Example 13 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ControllerEventProcessorTest method setUp.

@Before
public void setUp() throws Exception {
    executor = Executors.newScheduledThreadPool(10);
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new RetryOneTime(2000));
    zkClient.start();
    streamStore = StreamStoreFactory.createZKStore(zkClient, executor);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, TaskStoreFactory.createInMemoryStore(executor), segmentHelperMock, executor, "1", mock(ConnectionFactory.class), false, "");
    // region createStream
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    streamStore.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, STREAM, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
// endregion
}

Example 14 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataStoreTest method sizeTest.

@Test
public void sizeTest() throws Exception {
    final String scope = "ScopeSize";
    final String stream = "StreamSize";
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final RetentionPolicy retentionPolicy = RetentionPolicy.builder().retentionType(RetentionPolicy.RetentionType.SIZE).retentionParam(100L).build();
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(policy).retentionPolicy(retentionPolicy).build();
    long start = System.currentTimeMillis();
    store.createScope(scope).get();
    store.createStream(scope, stream, configuration, start, null, executor).get();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    store.addUpdateStreamForAutoStreamCut(scope, stream, retentionPolicy, null, executor).get();
    List<String> streams = store.getStreamsForBucket(0, executor).get();
    assertTrue(streams.contains(String.format("%s/%s", scope, stream)));
    // region Size Computation on stream cuts on epoch 0
    Map<Integer, Long> map1 = new HashMap<>();
    map1.put(0, 10L);
    map1.put(1, 10L);
    Long size = store.getSizeTillStreamCut(scope, stream, map1, null, executor).join();
    assertTrue(size == 20L);
    long recordingTime = System.currentTimeMillis();
    StreamCutRecord streamCut1 = new StreamCutRecord(recordingTime, size, map1);
    store.addStreamCutToRetentionSet(scope, stream, streamCut1, null, executor).get();
    Map<Integer, Long> map2 = new HashMap<>();
    map2.put(0, 20L);
    map2.put(1, 20L);
    size = store.getSizeTillStreamCut(scope, stream, map2, null, executor).join();
    assertTrue(size == 40L);
    StreamCutRecord streamCut2 = new StreamCutRecord(recordingTime + 10, size, map2);
    store.addStreamCutToRetentionSet(scope, stream, streamCut2, null, executor).get();
    Map<Integer, Long> map3 = new HashMap<>();
    map3.put(0, 30L);
    map3.put(1, 30L);
    size = store.getSizeTillStreamCut(scope, stream, map3, null, executor).join();
    assertTrue(size == 60L);
    StreamCutRecord streamCut3 = new StreamCutRecord(recordingTime + 20, 60L, map3);
    store.addStreamCutToRetentionSet(scope, stream, streamCut3, null, executor).get();
    // endregion
    // region Size Computation on multiple epochs
    long scaleTs = System.currentTimeMillis();
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.0, 0.5);
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.5, 1.0);
    List<Integer> scale1SealedSegments = Lists.newArrayList(0, 1);
    StartScaleResponse response = store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment2, segment3), scaleTs, false, null, executor).join();
    final List<Segment> scale1SegmentsCreated = response.getSegmentsCreated();
    store.setState(scope, stream, State.SCALING, null, executor).get();
    store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 40L)), scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    // complex stream cut - across two epochs
    Map<Integer, Long> map4 = new HashMap<>();
    map4.put(0, 40L);
    map4.put(3, 10L);
    size = store.getSizeTillStreamCut(scope, stream, map4, null, executor).join();
    assertTrue(size == 90L);
    StreamCutRecord streamCut4 = new StreamCutRecord(recordingTime + 30, size, map4);
    store.addStreamCutToRetentionSet(scope, stream, streamCut4, null, executor).get();
    // simple stream cut on epoch 2
    Map<Integer, Long> map5 = new HashMap<>();
    map5.put(2, 10L);
    map5.put(3, 10L);
    size = store.getSizeTillStreamCut(scope, stream, map5, null, executor).join();
    assertTrue(size == 100L);
    StreamCutRecord streamCut5 = new StreamCutRecord(recordingTime + 30, size, map5);
    store.addStreamCutToRetentionSet(scope, stream, streamCut5, null, executor).get();
// endregion
}

Example 15 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataStoreTest method scaleTest.

@Test
public void scaleTest() throws Exception {
    final String scope = "ScopeScale";
    final String stream = "StreamScale";
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(policy).build();
    long start = System.currentTimeMillis();
    store.createScope(scope).get();
    store.createStream(scope, stream, configuration, start, null, executor).get();
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
    // region idempotent
    long scaleTs = System.currentTimeMillis();
    SimpleEntry<Double, Double> segment1 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment2 = new SimpleEntry<>(0.75, 1.0);
    List<Integer> scale1SealedSegments = Collections.singletonList(1);
    // test run only if started
    AssertExtensions.assertThrows("", () -> store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, true, null, executor).join(), e -> Exceptions.unwrap(e) instanceof TaskExceptions.StartException);
    // 1. start scale
    StartScaleResponse response = store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, false, null, executor).join();
    final List<Segment> scale1SegmentsCreated = response.getSegmentsCreated();
    final int scale1ActiveEpoch = response.getActiveEpoch();
    // rerun start scale
    response = store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, false, null, executor).join();
    assertEquals(response.getSegmentsCreated(), scale1SegmentsCreated);
    store.setState(scope, stream, State.SCALING, null, executor).get();
    // 2. scale new segments created
    store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    // rerun start scale and new segments created
    response = store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, false, null, executor).join();
    assertEquals(response.getSegmentsCreated(), scale1SegmentsCreated);
    store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    // 3. scale segments sealed -- this will complete scale
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), scale1SegmentsCreated, response.getActiveEpoch(), scaleTs, null, executor).join();
    // rerun -- illegal state exception
    AssertExtensions.assertThrows("", () -> store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, scale1ActiveEpoch, scaleTs, null, executor).join(), e -> Exceptions.unwrap(e) instanceof StoreException.IllegalStateException);
    // rerun  -- illegal state exception
    AssertExtensions.assertThrows("", () -> store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), scale1SegmentsCreated, scale1ActiveEpoch, scaleTs, null, executor).join(), e -> Exceptions.unwrap(e) instanceof StoreException.IllegalStateException);
    // rerun start scale -- should fail with precondition failure
    AssertExtensions.assertThrows("", () -> store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, false, null, executor).join(), e -> Exceptions.unwrap(e) instanceof ScaleOperationExceptions.ScalePreConditionFailureException);
    // endregion
    // 2 different conflicting scale operations
    // region run concurrent conflicting scale
    SimpleEntry<Double, Double> segment3 = new SimpleEntry<>(0.0, 0.5);
    SimpleEntry<Double, Double> segment4 = new SimpleEntry<>(0.5, 0.75);
    SimpleEntry<Double, Double> segment5 = new SimpleEntry<>(0.75, 1.0);
    List<Integer> scale2SealedSegments = Arrays.asList(0, 2, 3);
    long scaleTs2 = System.currentTimeMillis();
    response = store.startScale(scope, stream, scale2SealedSegments, Arrays.asList(segment3, segment4, segment5), scaleTs2, false, null, executor).get();
    final List<Segment> scale2SegmentsCreated = response.getSegmentsCreated();
    final int scale2ActiveEpoch = response.getActiveEpoch();
    store.setState(scope, stream, State.SCALING, null, executor).get();
    // rerun of scale 1 -- should fail with precondition failure
    AssertExtensions.assertThrows("", () -> store.startScale(scope, stream, scale1SealedSegments, Arrays.asList(segment1, segment2), scaleTs, false, null, executor).join(), e -> Exceptions.unwrap(e) instanceof ScaleOperationExceptions.ScaleConflictException);
    // rerun of scale 1's new segments created method
    AssertExtensions.assertThrows("", () -> store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, scale1ActiveEpoch, scaleTs, null, executor).join(), e -> Exceptions.unwrap(e) instanceof ScaleOperationExceptions.ScaleConditionInvalidException);
    store.scaleNewSegmentsCreated(scope, stream, scale2SealedSegments, scale2SegmentsCreated, scale2ActiveEpoch, scaleTs2, null, executor).get();
    // rerun of scale 1's new segments created method
    AssertExtensions.assertThrows("", () -> store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, scale1ActiveEpoch, scaleTs, null, executor).join(), e -> Exceptions.unwrap(e) instanceof ScaleOperationExceptions.ScaleConditionInvalidException);
    store.scaleSegmentsSealed(scope, stream, scale1SealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), scale1SegmentsCreated, scale2ActiveEpoch, scaleTs2, null, executor).get();
    store.setState(scope, stream, State.SCALING, null, executor).get();
    // rerun of scale 1's new segments created method
    AssertExtensions.assertThrows("", () -> store.scaleNewSegmentsCreated(scope, stream, scale1SealedSegments, scale1SegmentsCreated, scale1ActiveEpoch, scaleTs, null, executor).join(), e -> Exceptions.unwrap(e) instanceof ScaleOperationExceptions.ScaleConditionInvalidException);
    store.setState(scope, stream, State.ACTIVE, null, executor).get();
// endregion
}