Search in sources :

Example 1 with AutoScaleEvent

use of io.pravega.shared.controller.event.AutoScaleEvent in project pravega by pravega.

the class AutoScaleProcessor method triggerScaleDown.

private void triggerScaleDown(String streamSegmentName, boolean silent) {
    if (initialized.get()) {
        Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
        long lastRequestTs = 0;
        if (pair != null && pair.getValue() != null) {
            lastRequestTs = pair.getValue();
        }
        long timestamp = System.currentTimeMillis();
        if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
            log.info("sending request for scale down for {}", streamSegmentName);
            Segment segment = Segment.fromScopedName(streamSegmentName);
            AutoScaleEvent event = new AutoScaleEvent(segment.getScope(), segment.getStreamName(), segment.getSegmentNumber(), AutoScaleEvent.DOWN, timestamp, 0, silent);
            writeRequest(event).thenAccept(x -> {
                if (!silent) {
                    // mute only scale downs
                    cache.put(streamSegmentName, new ImmutablePair<>(0L, timestamp));
                }
            });
        }
    }
}
Also used : Segment(io.pravega.client.segment.impl.Segment) AutoScaleEvent(io.pravega.shared.controller.event.AutoScaleEvent)

Example 2 with AutoScaleEvent

use of io.pravega.shared.controller.event.AutoScaleEvent in project pravega by pravega.

the class AutoScaleProcessor method triggerScaleUp.

private void triggerScaleUp(String streamSegmentName, int numOfSplits) {
    if (initialized.get()) {
        Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
        long lastRequestTs = 0;
        if (pair != null && pair.getKey() != null) {
            lastRequestTs = pair.getKey();
        }
        long timestamp = System.currentTimeMillis();
        if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
            log.info("sending request for scale up for {}", streamSegmentName);
            Segment segment = Segment.fromScopedName(streamSegmentName);
            AutoScaleEvent event = new AutoScaleEvent(segment.getScope(), segment.getStreamName(), segment.getSegmentNumber(), AutoScaleEvent.UP, timestamp, numOfSplits, false);
            // Mute scale for timestamp for both scale up and down
            writeRequest(event).thenAccept(x -> cache.put(streamSegmentName, new ImmutablePair<>(timestamp, timestamp)));
        }
    }
}
Also used : ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) Segment(io.pravega.client.segment.impl.Segment) AutoScaleEvent(io.pravega.shared.controller.event.AutoScaleEvent)

Example 3 with AutoScaleEvent

use of io.pravega.shared.controller.event.AutoScaleEvent in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequest.

@Test(timeout = 20000)
public void testScaleRequest() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, executor);
    // Send number of splits = 1
    AutoScaleEvent request = new AutoScaleEvent(scope, stream, 2, AutoScaleEvent.UP, System.currentTimeMillis(), 1, false);
    CompletableFuture<ScaleOpEvent> request1 = new CompletableFuture<>();
    CompletableFuture<ScaleOpEvent> request2 = new CompletableFuture<>();
    EventStreamWriter<ControllerEvent> writer = createWriter(x -> {
        if (!request1.isDone()) {
            final ArrayList<AbstractMap.SimpleEntry<Double, Double>> expected = new ArrayList<>();
            double start = 2.0 / 3.0;
            double end = 1.0;
            double middle = (start + end) / 2;
            expected.add(new AbstractMap.SimpleEntry<>(start, middle));
            expected.add(new AbstractMap.SimpleEntry<>(middle, end));
            checkRequest(request1, x, Lists.newArrayList(2), expected);
        } else if (!request2.isDone()) {
            final ArrayList<AbstractMap.SimpleEntry<Double, Double>> expected = new ArrayList<>();
            double start = 2.0 / 3.0;
            double end = 1.0;
            expected.add(new AbstractMap.SimpleEntry<>(start, end));
            checkRequest(request2, x, Lists.newArrayList(3, 4), expected);
        }
    });
    when(clientFactory.createEventWriter(eq(Config.SCALE_STREAM_NAME), eq(new JavaSerializer<ControllerEvent>()), any())).thenReturn(writer);
    assertTrue(Futures.await(multiplexer.process(request)));
    assertTrue(Futures.await(request1));
    assertTrue(Futures.await(multiplexer.process(request1.get())));
    // verify that the event is posted successfully
    List<Segment> activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 2));
    // verify that two splits are created even when we sent 1 as numOfSplits in AutoScaleEvent.
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.size() == 4);
    request = new AutoScaleEvent(scope, stream, 4, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false);
    assertTrue(Futures.await(multiplexer.process(request)));
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.size() == 4);
    request = new AutoScaleEvent(scope, stream, 3, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false);
    assertTrue(Futures.await(multiplexer.process(request)));
    assertTrue(Futures.await(request2));
    assertTrue(Futures.await(multiplexer.process(request2.get())));
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 5));
    assertTrue(activeSegments.size() == 3);
    // make it throw a non retryable failure so that test does not wait for number of retries.
    // This will bring down the test duration drastically because a retryable failure can keep retrying for few seconds.
    // And if someone changes retry durations and number of attempts in retry helper, it will impact this test's running time.
    // hence sending incorrect segmentsToSeal list which will result in a non retryable failure and this will fail immediately
    assertFalse(Futures.await(multiplexer.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(6), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), true, System.currentTimeMillis()))));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 5));
    assertTrue(activeSegments.size() == 3);
    assertFalse(Futures.await(multiplexer.process(new AbortEvent(scope, stream, 0, UUID.randomUUID()))));
}
Also used : EventStreamWriter(io.pravega.client.stream.EventStreamWriter) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) InetAddress(java.net.InetAddress) AutoScaleTask(io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask) TaskMetadataStore(io.pravega.controller.store.task.TaskMetadataStore) After(org.junit.After) Transaction(io.pravega.client.stream.Transaction) UUID(java.util.UUID) ScaleOperationTask(io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask) Sets(com.google.common.collect.Sets) Executors(java.util.concurrent.Executors) ControllerEvent(io.pravega.shared.controller.event.ControllerEvent) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) Config(io.pravega.controller.util.Config) Assert.assertFalse(org.junit.Assert.assertFalse) ClientFactory(io.pravega.client.ClientFactory) StreamMetadataStore(io.pravega.controller.store.stream.StreamMetadataStore) Futures(io.pravega.common.concurrent.Futures) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) CuratorFrameworkFactory(org.apache.curator.framework.CuratorFrameworkFactory) StreamStoreFactory(io.pravega.controller.store.stream.StreamStoreFactory) AutoScaleEvent(io.pravega.shared.controller.event.AutoScaleEvent) SegmentHelper(io.pravega.controller.server.SegmentHelper) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) AbortEvent(io.pravega.shared.controller.event.AbortEvent) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) TestingServerStarter(io.pravega.test.common.TestingServerStarter) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TestingServer(org.apache.curator.test.TestingServer) StreamMetadataTasks(io.pravega.controller.task.Stream.StreamMetadataTasks) Segment(io.pravega.controller.store.stream.Segment) HostMonitorConfigImpl(io.pravega.controller.store.host.impl.HostMonitorConfigImpl) Before(org.junit.Before) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) SegmentHelperMock(io.pravega.controller.mocks.SegmentHelperMock) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) UnknownHostException(java.net.UnknownHostException) HostStoreFactory(io.pravega.controller.store.host.HostStoreFactory) ExecutionException(java.util.concurrent.ExecutionException) Consumer(java.util.function.Consumer) AbstractMap(java.util.AbstractMap) TaskStoreFactory(io.pravega.controller.store.task.TaskStoreFactory) ConnectionFactoryImpl(io.pravega.client.netty.impl.ConnectionFactoryImpl) HostControllerStore(io.pravega.controller.store.host.HostControllerStore) StreamTransactionMetadataTasks(io.pravega.controller.task.Stream.StreamTransactionMetadataTasks) StreamRequestHandler(io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler) ScaleOpEvent(io.pravega.shared.controller.event.ScaleOpEvent) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) ClientConfig(io.pravega.client.ClientConfig) ArrayList(java.util.ArrayList) ScaleOperationTask(io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) AutoScaleEvent(io.pravega.shared.controller.event.AutoScaleEvent) ScaleOpEvent(io.pravega.shared.controller.event.ScaleOpEvent) Segment(io.pravega.controller.store.stream.Segment) ControllerEvent(io.pravega.shared.controller.event.ControllerEvent) AbstractMap(java.util.AbstractMap) AutoScaleTask(io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask) CompletableFuture(java.util.concurrent.CompletableFuture) StreamRequestHandler(io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler) AbortEvent(io.pravega.shared.controller.event.AbortEvent) Test(org.junit.Test)

Aggregations

AutoScaleEvent (io.pravega.shared.controller.event.AutoScaleEvent)3 Segment (io.pravega.client.segment.impl.Segment)2 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 ClientConfig (io.pravega.client.ClientConfig)1 ClientFactory (io.pravega.client.ClientFactory)1 ConnectionFactoryImpl (io.pravega.client.netty.impl.ConnectionFactoryImpl)1 EventStreamWriter (io.pravega.client.stream.EventStreamWriter)1 EventWriterConfig (io.pravega.client.stream.EventWriterConfig)1 ScalingPolicy (io.pravega.client.stream.ScalingPolicy)1 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)1 Transaction (io.pravega.client.stream.Transaction)1 JavaSerializer (io.pravega.client.stream.impl.JavaSerializer)1 Futures (io.pravega.common.concurrent.Futures)1 SegmentHelperMock (io.pravega.controller.mocks.SegmentHelperMock)1 SegmentHelper (io.pravega.controller.server.SegmentHelper)1 AutoScaleTask (io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask)1 ScaleOperationTask (io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask)1 StreamRequestHandler (io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler)1 HostControllerStore (io.pravega.controller.store.host.HostControllerStore)1