Examples with AutoScaleTask io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask used on opensource projects

Example 1 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ControllerServiceWithStreamTest method setup.

@Before
public void setup() {
    try {
        zkServer = new TestingServerStarter().start();
    } catch (Exception e) {
        log.error("Error starting ZK server", e);
    }
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    streamStore = spy(getStore());
    kvtStore = spy(getKVTStore());
    BucketStore bucketStore = StreamStoreFactory.createZKBucketStore(zkClient, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    GrpcAuthHelper disabledAuthHelper = GrpcAuthHelper.getDisabledAuthHelper();
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", disabledAuthHelper, helperMock);
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelperMock, executor, "host", disabledAuthHelper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
}

Example 2 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ControllerEventProcessorTest method testCommitAndStreamProcessorFairness.

@Test(timeout = 30000)
public void testCommitAndStreamProcessorFairness() {
    List<VersionedTransactionData> txnDataList1 = createAndCommitTransactions(3);
    int epoch = txnDataList1.get(0).getEpoch();
    streamStore.startCommitTransactions(SCOPE, STREAM, 100, null, executor).join();
    EventStreamWriterMock<ControllerEvent> requestEventWriter = new EventStreamWriterMock<>();
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    CommitRequestHandler commitEventProcessor = new CommitRequestHandler(streamStore, streamMetadataTasks, streamTransactionMetadataTasks, bucketStore, executor);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), null, null, null, null, null, null, null, streamStore, null, executor);
    // set some processor name so that the processing gets postponed
    streamStore.createWaitingRequestIfAbsent(SCOPE, STREAM, "test", null, executor).join();
    AssertExtensions.assertFutureThrows("Operation should be disallowed", commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStore.deleteWaitingRequestConditionally(SCOPE, STREAM, "test1", null, executor).join();
    assertEquals("test", streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
    // now remove the barrier but change the state so that processing can not happen.
    streamStore.deleteWaitingRequestConditionally(SCOPE, STREAM, "test", null, executor).join();
    assertNull(streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
    streamStore.setState(SCOPE, STREAM, State.SCALING, null, executor).join();
    AssertExtensions.assertFutureThrows("Operation should be disallowed", commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    assertEquals(commitEventProcessor.getProcessorName(), streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    // verify that we are able to process if the waiting processor name is same as ours.
    commitEventProcessor.processEvent(new CommitEvent(SCOPE, STREAM, epoch)).join();
    // verify that waiting processor is cleaned up.
    assertNull(streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
    // now set the state to COMMITTING_TXN and try the same with scaling
    streamStore.setState(SCOPE, STREAM, State.COMMITTING_TXN, null, executor).join();
    // verify that event that does not try to use `processor.withCompletion` runs without contention
    assertTrue(Futures.await(streamRequestHandler.processEvent(new AutoScaleEvent(SCOPE, STREAM, 0L, AutoScaleEvent.UP, 0L, 2, true, 0L))));
    // now same event's processing in face of a barrier should get postponed
    streamStore.createWaitingRequestIfAbsent(SCOPE, STREAM, commitEventProcessor.getProcessorName(), null, executor).join();
    assertTrue(Futures.await(streamRequestHandler.processEvent(new AutoScaleEvent(SCOPE, STREAM, 0L, AutoScaleEvent.UP, 0L, 2, true, 0L))));
    AssertExtensions.assertFutureThrows("Operation should be disallowed", streamRequestHandler.processEvent(new ScaleOpEvent(SCOPE, STREAM, Collections.emptyList(), Collections.emptyList(), false, 0L, 0L)), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    assertEquals(commitEventProcessor.getProcessorName(), streamStore.getWaitingRequestProcessor(SCOPE, STREAM, null, executor).join());
}

Example 3 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ControllerServiceWithKVTableTest method setup.

@Before
public void setup() {
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMockForTables(executor);
    streamStore = spy(getStore());
    kvtStore = spy(getKVTStore());
    BucketStore bucketStore = StreamStoreFactory.createZKBucketStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    GrpcAuthHelper disabledAuthHelper = GrpcAuthHelper.getDisabledAuthHelper();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", disabledAuthHelper, helperMock);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelperMock, executor, "host", disabledAuthHelper);
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
}

Example 4 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class RequestHandlersTest method testDeleteIgnoreFairness.

@Test
public void testDeleteIgnoreFairness() {
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtTasks, executor), executor);
    String fairness = "fairness";
    streamStore.createScope(fairness, null, executor).join();
    long createTimestamp = System.currentTimeMillis();
    streamMetadataTasks.createStream(fairness, fairness, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), createTimestamp, 0L).join();
    // 1. set segment helper mock to throw exception
    doAnswer(x -> Futures.failedFuture(new RuntimeException())).when(segmentHelper).deleteSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 2. start process --> this should fail with a retryable exception while talking to segment store!
    streamStore.setState(fairness, fairness, State.SEALED, null, executor).join();
    assertEquals(State.SEALED, streamStore.getState(fairness, fairness, true, null, executor).join());
    DeleteStreamEvent event = new DeleteStreamEvent(fairness, fairness, 0L, createTimestamp);
    AssertExtensions.assertFutureThrows("", streamRequestHandler.process(event, () -> false), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    verify(segmentHelper, atLeastOnce()).deleteSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 3. set waiting processor to "random name"
    streamStore.createWaitingRequestIfAbsent(fairness, fairness, "myProcessor", null, executor).join();
    // 4. reset segment helper to return success
    doAnswer(x -> CompletableFuture.completedFuture(null)).when(segmentHelper).deleteSegment(anyString(), anyString(), anyLong(), anyString(), anyLong());
    // 5. process again. it should succeed while ignoring waiting processor
    streamRequestHandler.process(event, () -> false).join();
    AssertExtensions.assertFutureThrows("", streamStore.getState(fairness, fairness, true, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
}

Example 5 with AutoScaleTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask in project pravega by pravega.

the class ControllerGrpcAuthFocusedTest method setup.

@Before
public void setup() throws IOException {
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createInMemoryStore(EXECUTOR);
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore();
    this.kvtStore = spy(KVTableStoreFactory.createInMemoryStore(streamStore, EXECUTOR));
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    SegmentHelper segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    RequestTracker requestTracker = new RequestTracker(true);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    GrpcAuthHelper authHelper = new GrpcAuthHelper(true, "secret", 300);
    EventHelper helper = EventHelperMock.getEventHelperMock(EXECUTOR, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelper, EXECUTOR, "host", authHelper, helper);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelper, EXECUTOR, "host", authHelper);
    kvtMetadataTasks = new TableMetadataTasks(kvtStore, segmentHelper, EXECUTOR, EXECUTOR, "host", authHelper, helper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, EXECUTOR), new ScaleOperationTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, EXECUTOR), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new TruncateStreamTask(streamMetadataTasks, streamStore, EXECUTOR), new CreateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, EXECUTOR), EXECUTOR);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, EXECUTOR));
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBase = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, true, 2);
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBaseStrict = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, false, 2);
    PasswordAuthHandler authHandler = new PasswordAuthHandler();
    authHandler.initialize(AUTH_FILE.getAbsolutePath());
    String uniqueServerName = String.format("Test server name: %s", getClass());
    String uniqueServerNameStrict = String.format("Test server name: %sStrict", getClass());
    // Using a builder that creates a server for servicing in-process requests.
    // Also, using a direct executor which executes app code directly in transport thread. See
    // https://grpc.io/grpc-java/javadoc/io/grpc/inprocess/InProcessServerBuilder.html for more information.
    grpcServer = InProcessServerBuilder.forName(uniqueServerName).addService(ServerInterceptors.intercept(controllerServiceImplBase, new AuthInterceptor(authHandler))).directExecutor().build().start();
    grpcServerStrict = InProcessServerBuilder.forName(uniqueServerNameStrict).addService(ServerInterceptors.intercept(controllerServiceImplBaseStrict, new AuthInterceptor(authHandler))).directExecutor().build().start();
    inProcessChannel = InProcessChannelBuilder.forName(uniqueServerName).directExecutor().build();
    inProcessChannelStrict = InProcessChannelBuilder.forName(uniqueServerNameStrict).directExecutor().build();
}