Search in sources :

Example 1 with JobStatusListener

use of org.apache.flink.runtime.executiongraph.JobStatusListener in project flink by apache.

the class DefaultSchedulerBatchSchedulingTest method testSchedulingOfJobWithFewerSlotsThanParallelism.

/**
 * Tests that a batch job can be executed with fewer slots than its parallelism. See FLINK-13187
 * for more information.
 */
@Test
public void testSchedulingOfJobWithFewerSlotsThanParallelism() throws Exception {
    final int parallelism = 5;
    final Time batchSlotTimeout = Time.milliseconds(5L);
    final JobGraph jobGraph = createBatchJobGraph(parallelism);
    try (final SlotPool slotPool = createSlotPool(mainThreadExecutor, batchSlotTimeout)) {
        final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(parallelism);
        TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((tdd, ignored) -> {
            submittedTasksQueue.offer(tdd.getExecutionAttemptId());
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).createTestingTaskExecutorGateway();
        final PhysicalSlotProvider slotProvider = new PhysicalSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
        final GloballyTerminalJobStatusListener jobStatusListener = new GloballyTerminalJobStatusListener();
        final SchedulerNG scheduler = createScheduler(jobGraph, mainThreadExecutor, slotProvider, batchSlotTimeout, jobStatusListener);
        CompletableFuture.runAsync(scheduler::startScheduling, mainThreadExecutor).join();
        // register a single slot at the slot pool
        SlotPoolUtils.offerSlots(slotPool, mainThreadExecutor, Collections.singletonList(ResourceProfile.ANY), new RpcTaskManagerGateway(testingTaskExecutorGateway, JobMasterId.generate()));
        // wait until the batch slot timeout has been reached
        Thread.sleep(batchSlotTimeout.toMilliseconds());
        final CompletableFuture<JobStatus> terminationFuture = jobStatusListener.getTerminationFuture();
        for (int i = 0; i < parallelism; i++) {
            final CompletableFuture<ExecutionAttemptID> submittedTaskFuture = CompletableFuture.supplyAsync(CheckedSupplier.unchecked(submittedTasksQueue::take));
            // wait until one of them is completed
            CompletableFuture.anyOf(submittedTaskFuture, terminationFuture).join();
            if (submittedTaskFuture.isDone()) {
                finishExecution(submittedTaskFuture.get(), scheduler, mainThreadExecutor);
            } else {
                fail(String.format("Job reached a globally terminal state %s before all executions were finished.", terminationFuture.get()));
            }
        }
        assertThat(terminationFuture.get(), is(JobStatus.FINISHED));
    }
}
Also used : ComponentMainThreadExecutorServiceAdapter(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter) TestingTaskExecutorGateway(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway) ComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) BeforeClass(org.junit.BeforeClass) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) DeclarativeSlotPoolBridgeBuilder(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeBuilder) SlotPoolUtils(org.apache.flink.runtime.jobmaster.slotpool.SlotPoolUtils) RpcTaskManagerGateway(org.apache.flink.runtime.jobmaster.RpcTaskManagerGateway) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) TestLogger(org.apache.flink.util.TestLogger) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Assert.fail(org.junit.Assert.fail) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) CheckedSupplier(org.apache.flink.util.function.CheckedSupplier) SlotPool(org.apache.flink.runtime.jobmaster.slotpool.SlotPool) AfterClass(org.junit.AfterClass) Logger(org.slf4j.Logger) PhysicalSlotProvider(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProvider) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) JobMasterId(org.apache.flink.runtime.jobmaster.JobMasterId) Test(org.junit.Test) LocationPreferenceSlotSelectionStrategy(org.apache.flink.runtime.jobmaster.slotpool.LocationPreferenceSlotSelectionStrategy) JobStatusListener(org.apache.flink.runtime.executiongraph.JobStatusListener) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) Executors(java.util.concurrent.Executors) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) PhysicalSlotProviderImpl(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProviderImpl) TestingTaskExecutorGatewayBuilder(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder) Matchers.is(org.hamcrest.Matchers.is) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) Collections(java.util.Collections) Time(org.apache.flink.api.common.time.Time) NoOpInvokable(org.apache.flink.runtime.testtasks.NoOpInvokable) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) RpcTaskManagerGateway(org.apache.flink.runtime.jobmaster.RpcTaskManagerGateway) Time(org.apache.flink.api.common.time.Time) TestingTaskExecutorGatewayBuilder(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder) SlotPool(org.apache.flink.runtime.jobmaster.slotpool.SlotPool) JobStatus(org.apache.flink.api.common.JobStatus) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) PhysicalSlotProviderImpl(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProviderImpl) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) TestingTaskExecutorGateway(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway) PhysicalSlotProvider(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProvider) Test(org.junit.Test)

Aggregations

Collections (java.util.Collections)1 ArrayBlockingQueue (java.util.concurrent.ArrayBlockingQueue)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 Executors (java.util.concurrent.Executors)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 JobStatus (org.apache.flink.api.common.JobStatus)1 Time (org.apache.flink.api.common.time.Time)1 ResourceProfile (org.apache.flink.runtime.clusterframework.types.ResourceProfile)1 ComponentMainThreadExecutor (org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor)1 ComponentMainThreadExecutorServiceAdapter (org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter)1 ExecutionState (org.apache.flink.runtime.execution.ExecutionState)1 ExecutionAttemptID (org.apache.flink.runtime.executiongraph.ExecutionAttemptID)1 JobStatusListener (org.apache.flink.runtime.executiongraph.JobStatusListener)1 JobGraph (org.apache.flink.runtime.jobgraph.JobGraph)1 JobGraphTestUtils (org.apache.flink.runtime.jobgraph.JobGraphTestUtils)1 JobVertex (org.apache.flink.runtime.jobgraph.JobVertex)1 JobMasterId (org.apache.flink.runtime.jobmaster.JobMasterId)1 RpcTaskManagerGateway (org.apache.flink.runtime.jobmaster.RpcTaskManagerGateway)1 DeclarativeSlotPoolBridgeBuilder (org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeBuilder)1 LocationPreferenceSlotSelectionStrategy (org.apache.flink.runtime.jobmaster.slotpool.LocationPreferenceSlotSelectionStrategy)1