Examples with JobStatus org.apache.flink.api.common.JobStatus used on opensource projects

Example 36 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class FileExecutionGraphInfoStoreTest method testStoredJobsOverview.

/**
 * Tests that we obtain the correct jobs overview.
 */
@Test
public void testStoredJobsOverview() throws IOException {
    final int numberExecutionGraphs = 10;
    final Collection<ExecutionGraphInfo> executionGraphInfos = generateTerminalExecutionGraphInfos(numberExecutionGraphs);
    final List<JobStatus> jobStatuses = executionGraphInfos.stream().map(ExecutionGraphInfo::getArchivedExecutionGraph).map(ArchivedExecutionGraph::getState).collect(Collectors.toList());
    final JobsOverview expectedJobsOverview = JobsOverview.create(jobStatuses);
    final File rootDir = temporaryFolder.newFolder();
    try (final FileExecutionGraphInfoStore executionGraphInfoStore = createDefaultExecutionGraphInfoStore(rootDir)) {
        for (ExecutionGraphInfo executionGraphInfo : executionGraphInfos) {
            executionGraphInfoStore.put(executionGraphInfo);
        }
        assertThat(executionGraphInfoStore.getStoredJobsOverview(), Matchers.equalTo(expectedJobsOverview));
    }
}

Example 37 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class AdaptiveScheduler method transitionToState.

// ----------------------------------------------------------------
/**
 * Transition the scheduler to another state. This method guards against state transitions while
 * there is already a transition ongoing. This effectively means that you can not call this
 * method from a State constructor or State#onLeave.
 *
 * @param targetState State to transition to
 * @param <T> Type of the target state
 * @return A target state instance
 */
@VisibleForTesting
<T extends State> T transitionToState(StateFactory<T> targetState) {
    Preconditions.checkState(!isTransitioningState, "State transitions must not be triggered while another state transition is in progress.");
    Preconditions.checkState(state.getClass() != targetState.getStateClass(), "Attempted to transition into the very state the scheduler is already in.");
    componentMainThreadExecutor.assertRunningInMainThread();
    try {
        isTransitioningState = true;
        LOG.debug("Transition from state {} to {}.", state.getClass().getSimpleName(), targetState.getStateClass().getSimpleName());
        final JobStatus previousJobStatus = state.getJobStatus();
        state.onLeave(targetState.getStateClass());
        T targetStateInstance = targetState.getState();
        state = targetStateInstance;
        final JobStatus newJobStatus = state.getJobStatus();
        if (previousJobStatus != newJobStatus) {
            final long timestamp = System.currentTimeMillis();
            jobStatusListeners.forEach(listener -> listener.jobStatusChanges(jobInformation.getJobID(), newJobStatus, timestamp));
        }
        return targetStateInstance;
    } finally {
        isTransitioningState = false;
    }
}

Example 38 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph.

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph is finished.
 */
@Test
public void testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph() throws Exception {
    final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
    CheckpointIDCounter counter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(counterShutdownFuture);
    final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
    CompletedCheckpointStore store = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(storeShutdownFuture);
    final SchedulerBase scheduler = createSchedulerAndEnableCheckpointing(counter, store);
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();
    assertThat(checkpointCoordinator, Matchers.notNullValue());
    assertThat(checkpointCoordinator.isShutdown(), is(false));
    scheduler.startScheduling();
    for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
        final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
        scheduler.updateTaskExecutionState(new TaskExecutionState(currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
    }
    assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));
    scheduler.closeAsync().get();
    assertThat(checkpointCoordinator.isShutdown(), is(true));
    assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
    assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}

Example 39 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method testClosingSchedulerShutsDownCheckpointCoordinatorOnSuspendedExecutionGraph.

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph is suspended.
 */
@Test
public void testClosingSchedulerShutsDownCheckpointCoordinatorOnSuspendedExecutionGraph() throws Exception {
    final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
    CheckpointIDCounter counter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(counterShutdownFuture);
    final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
    CompletedCheckpointStore store = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(storeShutdownFuture);
    final SchedulerBase scheduler = createSchedulerAndEnableCheckpointing(counter, store);
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();
    assertThat(checkpointCoordinator, Matchers.notNullValue());
    assertThat(checkpointCoordinator.isShutdown(), is(false));
    graph.suspend(new Exception("Test Exception"));
    scheduler.closeAsync().get();
    assertThat(checkpointCoordinator.isShutdown(), is(true));
    assertThat(counterShutdownFuture.get(), is(JobStatus.SUSPENDED));
    assertThat(storeShutdownFuture.get(), is(JobStatus.SUSPENDED));
}

Example 40 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class CompletedCheckpointTest method testCleanUpOnShutdown.

/**
 * Tests that the garbage collection properties are respected when shutting down.
 */
@Test
public void testCleanUpOnShutdown() throws Exception {
    JobStatus[] terminalStates = new JobStatus[] { JobStatus.FINISHED, JobStatus.CANCELED, JobStatus.FAILED, JobStatus.SUSPENDED };
    for (JobStatus status : terminalStates) {
        OperatorState state = mock(OperatorState.class);
        Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
        operatorStates.put(new OperatorID(), state);
        EmptyStreamStateHandle retainedHandle = new EmptyStreamStateHandle();
        TestCompletedCheckpointStorageLocation retainedLocation = new TestCompletedCheckpointStorageLocation(retainedHandle, "ptr");
        // Keep
        CheckpointProperties retainProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, false, false, false, false, false, false);
        CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, new HashMap<>(operatorStates), Collections.emptyList(), retainProps, retainedLocation);
        checkpoint.discardOnShutdown(status);
        verify(state, times(0)).discardState();
        assertFalse(retainedLocation.isDisposed());
        assertFalse(retainedHandle.isDisposed());
        // Discard
        EmptyStreamStateHandle discardHandle = new EmptyStreamStateHandle();
        TestCompletedCheckpointStorageLocation discardLocation = new TestCompletedCheckpointStorageLocation(discardHandle, "ptr");
        // Keep
        CheckpointProperties discardProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, true, true, true, true, true, false);
        checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, new HashMap<>(operatorStates), Collections.emptyList(), discardProps, discardLocation);
        checkpoint.discardOnShutdown(status);
        verify(state, times(1)).discardState();
        assertTrue(discardLocation.isDisposed());
        assertTrue(discardHandle.isDisposed());
    }
}