Examples with TestingTaskExecutorGateway org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway used on opensource projects

Example 36 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method registerTaskExecutorWithTwoSlots.

private void registerTaskExecutorWithTwoSlots(DeclarativeSlotManager slotManager, CompletableFuture<JobID> firstRequestSlotFuture) {
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple6 -> {
        firstRequestSlotFuture.complete(slotIDJobIDAllocationIDStringResourceManagerIdTuple6.f1);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection firstTaskExecutorConnection = createTaskExecutorConnection(taskExecutorGateway);
    final SlotReport firstSlotReport = createSlotReport(firstTaskExecutorConnection.getResourceID(), 2);
    slotManager.registerTaskManager(firstTaskExecutorConnection, firstSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
}

Example 37 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class DefaultSchedulerBatchSchedulingTest method testSchedulingOfJobWithFewerSlotsThanParallelism.

/**
 * Tests that a batch job can be executed with fewer slots than its parallelism. See FLINK-13187
 * for more information.
 */
@Test
public void testSchedulingOfJobWithFewerSlotsThanParallelism() throws Exception {
    final int parallelism = 5;
    final Time batchSlotTimeout = Time.milliseconds(5L);
    final JobGraph jobGraph = createBatchJobGraph(parallelism);
    try (final SlotPool slotPool = createSlotPool(mainThreadExecutor, batchSlotTimeout)) {
        final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(parallelism);
        TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((tdd, ignored) -> {
            submittedTasksQueue.offer(tdd.getExecutionAttemptId());
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).createTestingTaskExecutorGateway();
        final PhysicalSlotProvider slotProvider = new PhysicalSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
        final GloballyTerminalJobStatusListener jobStatusListener = new GloballyTerminalJobStatusListener();
        final SchedulerNG scheduler = createScheduler(jobGraph, mainThreadExecutor, slotProvider, batchSlotTimeout, jobStatusListener);
        CompletableFuture.runAsync(scheduler::startScheduling, mainThreadExecutor).join();
        // register a single slot at the slot pool
        SlotPoolUtils.offerSlots(slotPool, mainThreadExecutor, Collections.singletonList(ResourceProfile.ANY), new RpcTaskManagerGateway(testingTaskExecutorGateway, JobMasterId.generate()));
        // wait until the batch slot timeout has been reached
        Thread.sleep(batchSlotTimeout.toMilliseconds());
        final CompletableFuture<JobStatus> terminationFuture = jobStatusListener.getTerminationFuture();
        for (int i = 0; i < parallelism; i++) {
            final CompletableFuture<ExecutionAttemptID> submittedTaskFuture = CompletableFuture.supplyAsync(CheckedSupplier.unchecked(submittedTasksQueue::take));
            // wait until one of them is completed
            CompletableFuture.anyOf(submittedTaskFuture, terminationFuture).join();
            if (submittedTaskFuture.isDone()) {
                finishExecution(submittedTaskFuture.get(), scheduler, mainThreadExecutor);
            } else {
                fail(String.format("Job reached a globally terminal state %s before all executions were finished.", terminationFuture.get()));
            }
        }
        assertThat(terminationFuture.get(), is(JobStatus.FINISHED));
    }
}