Examples with JobStatus org.apache.flink.api.common.JobStatus used on opensource projects

Example 51 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class JobsOverview method create.

public static JobsOverview create(Collection<JobStatus> allJobsStatus) {
    Preconditions.checkNotNull(allJobsStatus);
    int numberRunningOrPendingJobs = 0;
    int numberFinishedJobs = 0;
    int numberCancelledJobs = 0;
    int numberFailedJobs = 0;
    for (JobStatus status : allJobsStatus) {
        switch(status) {
            case FINISHED:
                numberFinishedJobs++;
                break;
            case FAILED:
                numberFailedJobs++;
                break;
            case CANCELED:
                numberCancelledJobs++;
                break;
            default:
                numberRunningOrPendingJobs++;
                break;
        }
    }
    return new JobsOverview(numberRunningOrPendingJobs, numberFinishedJobs, numberCancelledJobs, numberFailedJobs);
}

Example 52 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class DefaultSchedulerBatchSchedulingTest method testSchedulingOfJobWithFewerSlotsThanParallelism.

/**
 * Tests that a batch job can be executed with fewer slots than its parallelism. See FLINK-13187
 * for more information.
 */
@Test
public void testSchedulingOfJobWithFewerSlotsThanParallelism() throws Exception {
    final int parallelism = 5;
    final Time batchSlotTimeout = Time.milliseconds(5L);
    final JobGraph jobGraph = createBatchJobGraph(parallelism);
    try (final SlotPool slotPool = createSlotPool(mainThreadExecutor, batchSlotTimeout)) {
        final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(parallelism);
        TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((tdd, ignored) -> {
            submittedTasksQueue.offer(tdd.getExecutionAttemptId());
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).createTestingTaskExecutorGateway();
        final PhysicalSlotProvider slotProvider = new PhysicalSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
        final GloballyTerminalJobStatusListener jobStatusListener = new GloballyTerminalJobStatusListener();
        final SchedulerNG scheduler = createScheduler(jobGraph, mainThreadExecutor, slotProvider, batchSlotTimeout, jobStatusListener);
        CompletableFuture.runAsync(scheduler::startScheduling, mainThreadExecutor).join();
        // register a single slot at the slot pool
        SlotPoolUtils.offerSlots(slotPool, mainThreadExecutor, Collections.singletonList(ResourceProfile.ANY), new RpcTaskManagerGateway(testingTaskExecutorGateway, JobMasterId.generate()));
        // wait until the batch slot timeout has been reached
        Thread.sleep(batchSlotTimeout.toMilliseconds());
        final CompletableFuture<JobStatus> terminationFuture = jobStatusListener.getTerminationFuture();
        for (int i = 0; i < parallelism; i++) {
            final CompletableFuture<ExecutionAttemptID> submittedTaskFuture = CompletableFuture.supplyAsync(CheckedSupplier.unchecked(submittedTasksQueue::take));
            // wait until one of them is completed
            CompletableFuture.anyOf(submittedTaskFuture, terminationFuture).join();
            if (submittedTaskFuture.isDone()) {
                finishExecution(submittedTaskFuture.get(), scheduler, mainThreadExecutor);
            } else {
                fail(String.format("Job reached a globally terminal state %s before all executions were finished.", terminationFuture.get()));
            }
        }
        assertThat(terminationFuture.get(), is(JobStatus.FINISHED));
    }
}

Example 53 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class DefaultSchedulerTest method failJobIfCannotRestart.

@Test
public void failJobIfCannotRestart() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    testRestartBackoffTimeStrategy.setCanRestart(false);
    final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
    final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices());
    final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(createFailedTaskExecutionState(attemptId));
    taskRestartExecutor.triggerScheduledTasks();
    waitForTermination(scheduler);
    final JobStatus jobStatus = scheduler.requestJobStatus();
    assertThat(jobStatus, is(equalTo(JobStatus.FAILED)));
}

Example 54 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class JobDetailsInfoTest method getTestResponseInstance.

@Override
protected JobDetailsInfo getTestResponseInstance() throws Exception {
    final Random random = new Random();
    final int numJobVertexDetailsInfos = 4;
    final String jsonPlan = "{\"id\":\"1234\"}";
    final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
    final Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(numJobVertexDetailsInfos);
    final Map<ExecutionState, Integer> jobVerticesPerState = new HashMap<>(ExecutionState.values().length);
    for (JobStatus jobStatus : JobStatus.values()) {
        timestamps.put(jobStatus, random.nextLong());
    }
    for (int i = 0; i < numJobVertexDetailsInfos; i++) {
        jobVertexInfos.add(createJobVertexDetailsInfo(random));
    }
    for (ExecutionState executionState : ExecutionState.values()) {
        jobVerticesPerState.put(executionState, random.nextInt());
    }
    return new JobDetailsInfo(new JobID(), "foobar", true, JobStatus.values()[random.nextInt(JobStatus.values().length)], 1L, 2L, 1L, 8888L, 1984L, timestamps, jobVertexInfos, jobVerticesPerState, jsonPlan);
}

Example 55 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class AdaptiveSchedulerTest method testCloseShutsDownCheckpointingComponents.

@Test
public void testCloseShutsDownCheckpointingComponents() throws Exception {
    final CompletableFuture<JobStatus> completedCheckpointStoreShutdownFuture = new CompletableFuture<>();
    final CompletedCheckpointStore completedCheckpointStore = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(completedCheckpointStoreShutdownFuture);
    final CompletableFuture<JobStatus> checkpointIdCounterShutdownFuture = new CompletableFuture<>();
    final CheckpointIDCounter checkpointIdCounter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(checkpointIdCounterShutdownFuture);
    final JobGraph jobGraph = createJobGraph();
    // checkpointing components are only created if checkpointing is enabled
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().build(), null));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setCheckpointRecoveryFactory(new TestingCheckpointRecoveryFactory(completedCheckpointStore, checkpointIdCounter)).build();
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        // transition into the FAILED state
        scheduler.handleGlobalFailure(new FlinkException("Test exception"));
        scheduler.closeAsync();
    });
    assertThat(completedCheckpointStoreShutdownFuture.get()).isEqualTo(JobStatus.FAILED);
    assertThat(checkpointIdCounterShutdownFuture.get()).isEqualTo(JobStatus.FAILED);
}