Examples with ScaleOperationTask io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask used on opensource projects

Example 11 with ScaleOperationTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask in project pravega by pravega.

the class ControllerGrpcAuthFocusedTest method setup.

@Before
public void setup() throws IOException {
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createInMemoryStore(EXECUTOR);
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore();
    this.kvtStore = spy(KVTableStoreFactory.createInMemoryStore(streamStore, EXECUTOR));
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    SegmentHelper segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    RequestTracker requestTracker = new RequestTracker(true);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    GrpcAuthHelper authHelper = new GrpcAuthHelper(true, "secret", 300);
    EventHelper helper = EventHelperMock.getEventHelperMock(EXECUTOR, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelper, EXECUTOR, "host", authHelper, helper);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelper, EXECUTOR, "host", authHelper);
    kvtMetadataTasks = new TableMetadataTasks(kvtStore, segmentHelper, EXECUTOR, EXECUTOR, "host", authHelper, helper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, EXECUTOR), new ScaleOperationTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, EXECUTOR), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new TruncateStreamTask(streamMetadataTasks, streamStore, EXECUTOR), new CreateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, EXECUTOR), EXECUTOR);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, EXECUTOR));
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBase = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, true, 2);
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBaseStrict = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, false, 2);
    PasswordAuthHandler authHandler = new PasswordAuthHandler();
    authHandler.initialize(AUTH_FILE.getAbsolutePath());
    String uniqueServerName = String.format("Test server name: %s", getClass());
    String uniqueServerNameStrict = String.format("Test server name: %sStrict", getClass());
    // Using a builder that creates a server for servicing in-process requests.
    // Also, using a direct executor which executes app code directly in transport thread. See
    // https://grpc.io/grpc-java/javadoc/io/grpc/inprocess/InProcessServerBuilder.html for more information.
    grpcServer = InProcessServerBuilder.forName(uniqueServerName).addService(ServerInterceptors.intercept(controllerServiceImplBase, new AuthInterceptor(authHandler))).directExecutor().build().start();
    grpcServerStrict = InProcessServerBuilder.forName(uniqueServerNameStrict).addService(ServerInterceptors.intercept(controllerServiceImplBaseStrict, new AuthInterceptor(authHandler))).directExecutor().build().start();
    inProcessChannel = InProcessChannelBuilder.forName(uniqueServerName).directExecutor().build();
    inProcessChannelStrict = InProcessChannelBuilder.forName(uniqueServerNameStrict).directExecutor().build();
}

Example 12 with ScaleOperationTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask in project pravega by pravega.

the class StreamMetadataTasksTest method truncateStreamTest.

@Test(timeout = 30000)
public void truncateStreamTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration = StreamConfiguration.builder().scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    assertNotEquals(0, consumer.getCurrentSegments(SCOPE, "test", 0L).get().size());
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    List<Map.Entry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 0.75));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.75, 1.0));
    ScaleResponse scaleOpResult = streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(1L), newRanges, 30, 0L).get();
    assertTrue(scaleOpResult.getStatus().equals(ScaleStreamStatus.STARTED));
    ScaleOperationTask scaleTask = new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor);
    assertTrue(Futures.await(scaleTask.execute((ScaleOpEvent) requestEventWriter.eventQueue.take())));
    // start truncation
    StreamTruncationRecord truncProp = streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor).join().getObject();
    assertFalse(truncProp.isUpdating());
    // 1. happy day test
    // update.. should succeed
    Map<Long, Long> streamCut = new HashMap<>();
    streamCut.put(0L, 1L);
    streamCut.put(1L, 11L);
    CompletableFuture<UpdateStreamStatus.Status> truncateFuture = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut, 0L);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    assertEquals(UpdateStreamStatus.Status.SUCCESS, truncateFuture.join());
    truncProp = streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor).join().getObject();
    assertTrue(truncProp.getStreamCut().equals(streamCut));
    assertTrue(truncProp.getStreamCut().equals(streamCut));
    // 2. change state to scaling
    streamStorePartialMock.setState(SCOPE, "test", State.SCALING, null, executor).get();
    // call update should fail without posting the event
    long two = NameUtils.computeSegmentId(2, 1);
    long three = NameUtils.computeSegmentId(3, 1);
    Map<Long, Long> streamCut2 = new HashMap<>();
    streamCut2.put(0L, 1L);
    streamCut2.put(two, 1L);
    streamCut2.put(three, 1L);
    streamMetadataTasks.truncateStream(SCOPE, "test", streamCut2, 0L);
    AtomicBoolean loop = new AtomicBoolean(false);
    Futures.loop(() -> !loop.get(), () -> Futures.delayedFuture(() -> streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor), 1000, executor).thenApply(x -> x.getObject().isUpdating()).thenAccept(loop::set), executor).join();
    // event posted, first step performed. now pick the event for processing
    TruncateStreamTask truncateStreamTask = new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    TruncateStreamEvent taken = (TruncateStreamEvent) requestEventWriter.eventQueue.take();
    AssertExtensions.assertFutureThrows("", truncateStreamTask.execute(taken), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    // now with state = active, process the same event. it should succeed now.
    assertTrue(Futures.await(truncateStreamTask.execute(taken)));
    // 3. multiple back to back updates.
    Map<Long, Long> streamCut3 = new HashMap<>();
    streamCut3.put(0L, 12L);
    streamCut3.put(two, 12L);
    streamCut3.put(three, 12L);
    CompletableFuture<UpdateStreamStatus.Status> truncateOp1 = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut3, 0L);
    // ensure that previous updatestream has posted the event and set status to updating,
    // only then call second updateStream
    AtomicBoolean loop2 = new AtomicBoolean(false);
    Futures.loop(() -> !loop2.get(), () -> streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor).thenApply(x -> x.getObject().isUpdating()).thenAccept(loop2::set), executor).join();
    truncProp = streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor).join().getObject();
    assertTrue(truncProp.getStreamCut().equals(streamCut3) && truncProp.isUpdating());
    // post the second update request. This should fail here itself as previous one has started.
    Map<Long, Long> streamCut4 = new HashMap<>();
    streamCut4.put(0L, 14L);
    streamCut4.put(two, 14L);
    streamCut4.put(three, 14L);
    CompletableFuture<UpdateStreamStatus.Status> truncateOpFuture2 = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut4, 0L);
    assertEquals(UpdateStreamStatus.Status.FAILURE, truncateOpFuture2.join());
    // process event
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    // verify that first request for update also completes with success.
    assertEquals(UpdateStreamStatus.Status.SUCCESS, truncateOp1.join());
    truncProp = streamStorePartialMock.getTruncationRecord(SCOPE, "test", null, executor).join().getObject();
    assertTrue(truncProp.getStreamCut().equals(streamCut3) && !truncProp.isUpdating());
    streamStorePartialMock.setState(SCOPE, "test", State.TRUNCATING, null, executor).join();
    TruncateStreamEvent event = new TruncateStreamEvent(SCOPE, "test", System.nanoTime());
    assertTrue(Futures.await(truncateStreamTask.execute(event)));
    // execute the event again. It should complete without doing anything.
    truncateStreamTask.execute(event).join();
    assertEquals(State.ACTIVE, streamStorePartialMock.getState(SCOPE, "test", true, null, executor).join());
    doReturn(CompletableFuture.completedFuture(true)).when(streamStorePartialMock).isScopeSealed(anyString(), any(), any());
    CompletableFuture<CreateStreamResponse> streamResponse = streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor);
    CreateStreamResponse.CreateStatus s = streamResponse.get().getStatus();
    assertEquals(CreateStreamResponse.CreateStatus.EXISTS_ACTIVE, streamResponse.get().getStatus());
}

Example 13 with ScaleOperationTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask in project pravega by pravega.

the class ControllerServiceWithZKStreamTest method setup.

@Before
public void setup() {
    try {
        zkServer = new TestingServerStarter().start();
    } catch (Exception e) {
        log.error("Error starting ZK server", e);
    }
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createZKStore(zkClient, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, executor), new SealStreamTask(streamMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
}

Example 14 with ScaleOperationTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask in project pravega by pravega.

the class InMemoryControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createInMemoryStore(executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createInMemoryStore(executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    controllerService = new ControllerServiceImpl(new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, mockCluster), "secret", false);
}

Example 15 with ScaleOperationTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask in project pravega by pravega.

the class ZKControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    final StreamMetadataStore streamStore;
    final HostControllerStore hostStore;
    final TaskMetadataStore taskMetadataStore;
    final SegmentHelper segmentHelper;
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    storeClient = StoreClientFactory.createZKStoreClient(zkClient);
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createStore(storeClient, executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createZKStore(zkClient, executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    cluster = new ClusterZKImpl(zkClient, ClusterType.CONTROLLER);
    final CountDownLatch latch = new CountDownLatch(1);
    cluster.addListener((type, host) -> latch.countDown());
    cluster.registerHost(new Host("localhost", 9090, null));
    latch.await();
    ControllerService controller = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, cluster);
    controllerService = new ControllerServiceImpl(controller, "", false);
}