Examples with BlockerSync org.apache.flink.core.testutils.BlockerSync used on opensource projects

Example 6 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class RestServerEndpointITCase method testShouldWaitForHandlersWhenClosing.

/**
 * Tests that after calling {@link RestServerEndpoint#closeAsync()}, the handlers are closed
 * first, and we wait for in-flight requests to finish. As long as not all handlers are closed,
 * HTTP requests should be served.
 */
@Test
public void testShouldWaitForHandlersWhenClosing() throws Exception {
    testHandler.closeFuture = new CompletableFuture<>();
    final BlockerSync sync = new BlockerSync();
    testHandler.handlerBody = id -> {
        // handlers where the CompletableFuture is finished by the RPC framework.
        return CompletableFuture.supplyAsync(() -> {
            try {
                sync.block();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
            return new TestResponse(id);
        });
    };
    // Initiate closing RestServerEndpoint but the test handler should block.
    final CompletableFuture<Void> closeRestServerEndpointFuture = serverEndpoint.closeAsync();
    assertThat(closeRestServerEndpointFuture.isDone(), is(false));
    // create an in-flight request
    final CompletableFuture<TestResponse> request = sendRequestToTestHandler(new TestRequest(1));
    sync.awaitBlocker();
    // Allow handler to close but there is still one in-flight request which should prevent
    // the RestServerEndpoint from closing.
    testHandler.closeFuture.complete(null);
    assertThat(closeRestServerEndpointFuture.isDone(), is(false));
    // Finish the in-flight request.
    sync.releaseBlocker();
    request.get(timeout.getSize(), timeout.getUnit());
    closeRestServerEndpointFuture.get(timeout.getSize(), timeout.getUnit());
}

Example 7 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor.

@Test
public void testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor() throws Exception {
    BlockerSync sync = new BlockerSync();
    ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {

        @Override
        public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
            sync.blockNonInterruptible();
            super.releasePartition(partitionId, cause);
        }
    };
    NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(blockingResultPartitionManager).setIoExecutor(TEST_EXECUTOR_SERVICE_RESOURCE.getExecutor()).build();
    final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
    final TaskExecutorPartitionTracker partitionTracker = new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {

        @Override
        public void startTrackingPartition(JobID producingJobId, TaskExecutorPartitionInfo partitionInfo) {
            super.startTrackingPartition(producingJobId, partitionInfo);
            startTrackingFuture.complete(partitionInfo.getResultPartitionId());
        }
    };
    try {
        internalTestPartitionRelease(partitionTracker, shuffleEnvironment, startTrackingFuture, (jobId, resultPartitionDeploymentDescriptor, taskExecutor, taskExecutorGateway) -> {
            final IntermediateDataSetID dataSetId = resultPartitionDeploymentDescriptor.getResultId();
            taskExecutorGateway.releaseClusterPartitions(Collections.singleton(dataSetId), timeout);
            // execute some operation to check whether the TaskExecutor is blocked
            taskExecutorGateway.canBeReleased().get(5, TimeUnit.SECONDS);
        });
    } finally {
        sync.releaseBlocker();
    }
}

Example 8 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class TaskExecutorExecutionDeploymentReconciliationTest method testDeployedExecutionReporting.

@Test
public void testDeployedExecutionReporting() throws Exception {
    final OneShotLatch slotOfferLatch = new OneShotLatch();
    final BlockingQueue<Set<ExecutionAttemptID>> deployedExecutionsQueue = new ArrayBlockingQueue<>(3);
    final CompletableFuture<Void> taskFinishedFuture = new CompletableFuture<>();
    final ResourceID jobManagerResourceId = ResourceID.generate();
    final TestingJobMasterGateway jobMasterGateway = setupJobManagerGateway(slotOfferLatch, deployedExecutionsQueue, taskFinishedFuture, jobManagerResourceId);
    final CompletableFuture<SlotReport> initialSlotReportFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway testingResourceManagerGateway = setupResourceManagerGateway(initialSlotReportFuture);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1, timeout)).setShuffleEnvironment(new NettyShuffleEnvironmentBuilder().build()).build();
    final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
    try {
        taskExecutor.start();
        taskExecutor.waitUntilStarted();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor(jobId);
        connectComponentsAndRequestSlot(jobMasterGateway, testingResourceManagerGateway, taskExecutorGateway, taskManagerServices.getJobLeaderService(), initialSlotReportFuture, taskDeploymentDescriptor.getAllocationId());
        TestingInvokable.sync = new BlockerSync();
        // This ensures TM has been successfully registered to JM.
        slotOfferLatch.await();
        AllocatedSlotReport slotAllocationReport = new AllocatedSlotReport(jobId, Collections.singleton(new AllocatedSlotInfo(0, taskDeploymentDescriptor.getAllocationId())));
        // nothing as deployed, so the deployment report should be empty
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasSize(0));
        taskExecutorGateway.submitTask(taskDeploymentDescriptor, jobMasterGateway.getFencingToken(), timeout).get();
        TestingInvokable.sync.awaitBlocker();
        // task is deployed, so the deployment report should contain it
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasItem(taskDeploymentDescriptor.getExecutionAttemptId()));
        TestingInvokable.sync.releaseBlocker();
        // task is finished ans was cleaned up, so the deployment report should be empty
        taskFinishedFuture.get();
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasSize(0));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}

Example 9 with BlockerSync

use of org.apache.flink.core.testutils.BlockerSync in project flink by apache.

the class NettyShuffleEnvironmentTest method testSlowIODoesNotBlockRelease.

@Test
public void testSlowIODoesNotBlockRelease() throws Exception {
    BlockerSync sync = new BlockerSync();
    ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {

        @Override
        public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
            sync.blockNonInterruptible();
            super.releasePartition(partitionId, cause);
        }
    };
    NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(blockingResultPartitionManager).setIoExecutor(Executors.newFixedThreadPool(1)).build();
    shuffleEnvironment.releasePartitionsLocally(Collections.singleton(new ResultPartitionID()));
    sync.awaitBlocker();
    sync.releaseBlocker();
}