Examples with AutoScaleTask io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask used on opensource projects

Example 11 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class RequestHandlersTest method testTruncateIgnoreFairness.

@Test
public void testTruncateIgnoreFairness() {
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtTasks, executor), executor);
    String fairness = "fairness";
    streamStore.createScope(fairness, null, executor).join();
    streamMetadataTasks.createStream(fairness, fairness, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), 0L).join();
    // 1. set segment helper mock to throw exception
    doAnswer(x -> Futures.failedFuture(new RuntimeException())).when(segmentHelper).truncateSegment(anyString(), anyString(), anyLong(), anyLong(), anyString(), anyLong());
    // 2. start process --> this should fail with a retryable exception while talking to segment store!
    streamStore.startTruncation(fairness, fairness, Collections.singletonMap(0L, 0L), null, executor).join();
    streamStore.setState(fairness, fairness, State.TRUNCATING, null, executor).join();
    assertEquals(State.TRUNCATING, streamStore.getState(fairness, fairness, true, null, executor).join());
    TruncateStreamEvent event = new TruncateStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event, () -> false), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    verify(segmentHelper, atLeastOnce()).truncateSegment(anyString(), anyString(), anyLong(), anyLong(), anyString(), anyLong());
    // 3. set waiting processor to "random name"
    streamStore.createWaitingRequestIfAbsent(fairness, fairness, "myProcessor", null, executor).join();
    // 4. reset segment helper to return success
    doAnswer(x -> CompletableFuture.completedFuture(null)).when(segmentHelper).truncateSegment(anyString(), anyString(), anyLong(), anyLong(), anyString(), anyLong());
    // 5. process again. it should succeed while ignoring waiting processor
    streamRequestHandler.process(event, () -> false).join();
    assertEquals(State.ACTIVE, streamStore.getState(fairness, fairness, true, null, executor).join());
    // 6. run a new update. it should fail because of waiting processor.
    TruncateStreamEvent event2 = new TruncateStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event2, () -> false), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStore.deleteWaitingRequestConditionally(fairness, fairness, "myProcessor", null, executor).join();
}

Example 12 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class RequestHandlersTest method testSealIgnoreFairness.

@Test
public void testSealIgnoreFairness() {
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtTasks, executor), executor);
    String fairness = "fairness";
    streamStore.createScope(fairness, null, executor).join();
    streamMetadataTasks.createStream(fairness, fairness, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), 0L).join();
    // 1. set segment helper mock to throw exception
    doAnswer(x -> Futures.failedFuture(new RuntimeException())).when(segmentHelper).sealSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 2. start process --> this should fail with a retryable exception while talking to segment store!
    streamStore.setState(fairness, fairness, State.SEALING, null, executor).join();
    assertEquals(State.SEALING, streamStore.getState(fairness, fairness, true, null, executor).join());
    SealStreamEvent event = new SealStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event, () -> false), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    verify(segmentHelper, atLeastOnce()).sealSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 3. set waiting processor to "random name"
    streamStore.createWaitingRequestIfAbsent(fairness, fairness, "myProcessor", null, executor).join();
    // 4. reset segment helper to return success
    doAnswer(x -> CompletableFuture.completedFuture(null)).when(segmentHelper).sealSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 5. process again. it should succeed while ignoring waiting processor
    streamRequestHandler.process(event, () -> false).join();
    assertEquals(State.SEALED, streamStore.getState(fairness, fairness, true, null, executor).join());
    // 6. run a new update. it should fail because of waiting processor.
    SealStreamEvent event2 = new SealStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event2, () -> false), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStore.deleteWaitingRequestConditionally(fairness, fairness, "myProcessor", null, executor).join();
}

Example 13 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class RequestHandlersTest method testUpdateIgnoreFairness.

@Test
public void testUpdateIgnoreFairness() {
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtTasks, executor), executor);
    String fairness = "fairness";
    streamStore.createScope(fairness, null, executor).join();
    streamMetadataTasks.createStream(fairness, fairness, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), 0L).join();
    // 1. set segment helper mock to throw exception
    doAnswer(x -> Futures.failedFuture(new RuntimeException())).when(segmentHelper).updatePolicy(anyString(), anyString(), any(), anyLong(), anyString(), anyLong());
    // 2. start process --> this should fail with a retryable exception while talking to segment store!
    streamStore.startUpdateConfiguration(fairness, fairness, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), null, executor).join();
    streamStore.setState(fairness, fairness, State.UPDATING, null, executor).join();
    assertEquals(State.UPDATING, streamStore.getState(fairness, fairness, true, null, executor).join());
    UpdateStreamEvent event = new UpdateStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event, () -> false), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    verify(segmentHelper, atLeastOnce()).updatePolicy(anyString(), anyString(), any(), anyLong(), anyString(), anyLong());
    // 3. set waiting processor to "random name"
    streamStore.createWaitingRequestIfAbsent(fairness, fairness, "myProcessor", null, executor).join();
    // 4. reset segment helper to return success
    doAnswer(x -> CompletableFuture.completedFuture(null)).when(segmentHelper).updatePolicy(anyString(), anyString(), any(), anyLong(), anyString(), anyLong());
    // 5. process again. it should succeed while ignoring waiting processor
    streamRequestHandler.process(event, () -> false).join();
    assertEquals(State.ACTIVE, streamStore.getState(fairness, fairness, true, null, executor).join());
    // 6. run a new update. it should fail because of waiting processor and our state does not allow us to ignore waiting processor
    UpdateStreamEvent event2 = new UpdateStreamEvent(fairness, fairness, 0L);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event2, () -> false), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStore.deleteWaitingRequestConditionally(fairness, fairness, "myProcessor", null, executor).join();
}

Example 14 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRange.

@Test(timeout = 30000)
public void testScaleRange() throws ExecutionException, InterruptedException {
    // key range values taken from issue #2543
    StreamSegmentRecord segment = new StreamSegmentRecord(2, 1, 100L, 0.1706574888245243, 0.7085170563088633);
    doReturn(CompletableFuture.completedFuture(segment)).when(streamStore).getSegment(any(), any(), anyLong(), any(), any());
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    // Send number of splits = 1
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    AutoScaleEvent scaleUpEvent = new AutoScaleEvent(scope, stream, NameUtils.computeSegmentId(2, 1), AutoScaleEvent.UP, System.currentTimeMillis(), 1, false, System.currentTimeMillis());
    assertTrue(Futures.await(multiplexer.process(scaleUpEvent, () -> false)));
    reset(streamStore);
    // verify that one scaleOp event is written into the stream
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleOpEvent = (ScaleOpEvent) event;
    assertEquals(2, scaleOpEvent.getNewRanges().size());
    assertEquals(0.1706574888245243, scaleOpEvent.getNewRanges().get(0).getKey(), 0.0);
    assertEquals(0.7085170563088633, scaleOpEvent.getNewRanges().get(1).getValue(), 0.0);
    assertTrue(scaleOpEvent.getNewRanges().get(0).getValue().doubleValue() == scaleOpEvent.getNewRanges().get(1).getKey().doubleValue());
}

Example 15 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequestWithMinimumSegment.

@Test(timeout = 30000)
public void testScaleRequestWithMinimumSegment() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    String stream = "mystream";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 5)).build();
    streamMetadataTasks.createStream(scope, stream, config, System.currentTimeMillis(), 0L).get();
    // change stream configuration to min segment count = 4
    config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(1, 2, 4)).build();
    streamStore.startUpdateConfiguration(scope, stream, config, null, executor).join();
    VersionedMetadata<StreamConfigurationRecord> configRecord = streamStore.getConfigurationRecord(scope, stream, null, executor).join();
    streamStore.completeUpdateConfiguration(scope, stream, configRecord, null, executor).join();
    // process first auto scale down event. it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 1L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(writer.queue.isEmpty());
    assertTrue(streamStore.isCold(scope, stream, 1L, null, executor).join());
    // process second auto scale down event. since its not for an immediate neighbour so it should only mark the segment as cold
    multiplexer.process(new AutoScaleEvent(scope, stream, 3L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 3L, null, executor).join());
    // no scale event should be posted
    assertTrue(writer.queue.isEmpty());
    // process third auto scale down event. This should result in a scale op event being posted to merge segments 0, 1
    multiplexer.process(new AutoScaleEvent(scope, stream, 0L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 0L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent1 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent1.getNewRanges().size());
    assertEquals(2, scaleDownEvent1.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(0L));
    assertTrue(scaleDownEvent1.getSegmentsToSeal().contains(1L));
    // process fourth auto scale down event. This should result in a scale op event being posted to merge segments 3, 4
    multiplexer.process(new AutoScaleEvent(scope, stream, 4L, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis()), () -> false).join();
    assertTrue(streamStore.isCold(scope, stream, 4L, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleDownEvent2 = (ScaleOpEvent) event;
    assertEquals(1, scaleDownEvent2.getNewRanges().size());
    assertEquals(2, scaleDownEvent2.getSegmentsToSeal().size());
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(3L));
    assertTrue(scaleDownEvent2.getSegmentsToSeal().contains(4L));
    // process first scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent1, () -> false).join();
    EpochRecord activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    List<StreamSegmentRecord> segments = activeEpoch.getSegments();
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
    // process second scale down event, this should submit scale and scale the stream down to 4 segments
    multiplexer.process(scaleDownEvent2, () -> false).join();
    // verify that no scale has happened
    activeEpoch = streamStore.getActiveEpoch(scope, stream, null, true, executor).join();
    // verify that no scale has happened.
    assertEquals(1, activeEpoch.getEpoch());
    assertEquals(4, segments.size());
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 2));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 3));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 4));
    assertTrue(segments.stream().anyMatch(x -> x.getSegmentNumber() == 5));
}