Examples with StreamRequestHandler io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler used on opensource projects

Example 1 with StreamRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequest.

@Test(timeout = 20000)
public void testScaleRequest() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, executor);
    // Send number of splits = 1
    AutoScaleEvent request = new AutoScaleEvent(scope, stream, 2, AutoScaleEvent.UP, System.currentTimeMillis(), 1, false);
    CompletableFuture<ScaleOpEvent> request1 = new CompletableFuture<>();
    CompletableFuture<ScaleOpEvent> request2 = new CompletableFuture<>();
    EventStreamWriter<ControllerEvent> writer = createWriter(x -> {
        if (!request1.isDone()) {
            final ArrayList<AbstractMap.SimpleEntry<Double, Double>> expected = new ArrayList<>();
            double start = 2.0 / 3.0;
            double end = 1.0;
            double middle = (start + end) / 2;
            expected.add(new AbstractMap.SimpleEntry<>(start, middle));
            expected.add(new AbstractMap.SimpleEntry<>(middle, end));
            checkRequest(request1, x, Lists.newArrayList(2), expected);
        } else if (!request2.isDone()) {
            final ArrayList<AbstractMap.SimpleEntry<Double, Double>> expected = new ArrayList<>();
            double start = 2.0 / 3.0;
            double end = 1.0;
            expected.add(new AbstractMap.SimpleEntry<>(start, end));
            checkRequest(request2, x, Lists.newArrayList(3, 4), expected);
        }
    });
    when(clientFactory.createEventWriter(eq(Config.SCALE_STREAM_NAME), eq(new JavaSerializer<ControllerEvent>()), any())).thenReturn(writer);
    assertTrue(Futures.await(multiplexer.process(request)));
    assertTrue(Futures.await(request1));
    assertTrue(Futures.await(multiplexer.process(request1.get())));
    // verify that the event is posted successfully
    List<Segment> activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 2));
    // verify that two splits are created even when we sent 1 as numOfSplits in AutoScaleEvent.
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.size() == 4);
    request = new AutoScaleEvent(scope, stream, 4, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false);
    assertTrue(Futures.await(multiplexer.process(request)));
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.size() == 4);
    request = new AutoScaleEvent(scope, stream, 3, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false);
    assertTrue(Futures.await(multiplexer.process(request)));
    assertTrue(Futures.await(request2));
    assertTrue(Futures.await(multiplexer.process(request2.get())));
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 5));
    assertTrue(activeSegments.size() == 3);
    // make it throw a non retryable failure so that test does not wait for number of retries.
    // This will bring down the test duration drastically because a retryable failure can keep retrying for few seconds.
    // And if someone changes retry durations and number of attempts in retry helper, it will impact this test's running time.
    // hence sending incorrect segmentsToSeal list which will result in a non retryable failure and this will fail immediately
    assertFalse(Futures.await(multiplexer.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(6), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.0, 1.0)), true, System.currentTimeMillis()))));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 3));
    assertTrue(activeSegments.stream().noneMatch(z -> z.getNumber() == 4));
    assertTrue(activeSegments.stream().anyMatch(z -> z.getNumber() == 5));
    assertTrue(activeSegments.size() == 3);
    assertFalse(Futures.await(multiplexer.process(new AbortEvent(scope, stream, 0, UUID.randomUUID()))));
}

Example 2 with StreamRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore(1, executor);
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    doReturn(CompletableFuture.completedFuture(false)).when(streamStorePartialMock).isTransactionOngoing(anyString(), anyString(), any(), // mock only isTransactionOngoing call.
    any());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key"));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, hostStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new SealStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), executor);
    consumer = new ControllerService(streamStorePartialMock, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStorePartialMock.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
}

Example 3 with StreamRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler in project pravega by pravega.

the class ControllerServiceWithZKStreamTest method setup.

@Before
public void setup() {
    try {
        zkServer = new TestingServerStarter().start();
    } catch (Exception e) {
        log.error("Error starting ZK server", e);
    }
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createZKStore(zkClient, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, executor), new SealStreamTask(streamMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
}

Example 4 with StreamRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler in project pravega by pravega.

the class InMemoryControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createInMemoryStore(executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createInMemoryStore(executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    controllerService = new ControllerServiceImpl(new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, mockCluster), "secret", false);
}

Example 5 with StreamRequestHandler

use of io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler in project pravega by pravega.

the class ZKControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    final StreamMetadataStore streamStore;
    final HostControllerStore hostStore;
    final TaskMetadataStore taskMetadataStore;
    final SegmentHelper segmentHelper;
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    storeClient = StoreClientFactory.createZKStoreClient(zkClient);
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createStore(storeClient, executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createZKStore(zkClient, executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    cluster = new ClusterZKImpl(zkClient, ClusterType.CONTROLLER);
    final CountDownLatch latch = new CountDownLatch(1);
    cluster.addListener((type, host) -> latch.countDown());
    cluster.registerHost(new Host("localhost", 9090, null));
    latch.await();
    ControllerService controller = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, cluster);
    controllerService = new ControllerServiceImpl(controller, "", false);
}