Examples with Execution org.apache.flink.runtime.executiongraph.Execution used on opensource projects

Example 36 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class ExecutionGraphHandler method requestNextInputSplit.

public SerializedInputSplit requestNextInputSplit(JobVertexID vertexID, ExecutionAttemptID executionAttempt) throws IOException {
    final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt);
    if (execution == null) {
        // but TaskManager get some delay to aware of that situation
        if (log.isDebugEnabled()) {
            log.debug("Can not find Execution for attempt {}.", executionAttempt);
        }
        // but we should TaskManager be aware of this
        throw new IllegalArgumentException("Can not find Execution for attempt " + executionAttempt);
    }
    final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID);
    if (vertex == null) {
        throw new IllegalArgumentException("Cannot find execution vertex for vertex ID " + vertexID);
    }
    if (vertex.getSplitAssigner() == null) {
        throw new IllegalStateException("No InputSplitAssigner for vertex ID " + vertexID);
    }
    final InputSplit nextInputSplit = execution.getNextInputSplit();
    if (nextInputSplit != null) {
        log.debug("Send next input split {}.", nextInputSplit);
    } else {
        log.debug("No more input splits available");
    }
    try {
        final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit);
        return new SerializedInputSplit(serializedInputSplit);
    } catch (Exception ex) {
        IOException reason = new IOException("Could not serialize the next input split of class " + nextInputSplit.getClass() + ".", ex);
        vertex.fail(reason);
        throw reason;
    }
}

Example 37 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph.

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph is finished.
 */
@Test
public void testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph() throws Exception {
    final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
    CheckpointIDCounter counter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(counterShutdownFuture);
    final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
    CompletedCheckpointStore store = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(storeShutdownFuture);
    final SchedulerBase scheduler = createSchedulerAndEnableCheckpointing(counter, store);
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();
    assertThat(checkpointCoordinator, Matchers.notNullValue());
    assertThat(checkpointCoordinator.isShutdown(), is(false));
    scheduler.startScheduling();
    for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
        final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
        scheduler.updateTaskExecutionState(new TaskExecutionState(currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
    }
    assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));
    scheduler.closeAsync().get();
    assertThat(checkpointCoordinator.isShutdown(), is(true));
    assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
    assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}

Example 38 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class CheckpointStateRestoreTest method testSetState.

/**
 * Tests that on restore the task state is reset for each stateful task.
 */
@Test
public void testSetState() {
    try {
        KeyGroupRange keyGroupRange = KeyGroupRange.of(0, 0);
        List<SerializableObject> testStates = Collections.singletonList(new SerializableObject());
        final KeyedStateHandle serializedKeyGroupStates = CheckpointCoordinatorTestingUtils.generateKeyGroupState(keyGroupRange, testStates);
        final JobVertexID statefulId = new JobVertexID();
        final JobVertexID statelessId = new JobVertexID();
        ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(statefulId, 3, 256).addJobVertex(statelessId, 2, 256).build();
        ExecutionJobVertex stateful = graph.getJobVertex(statefulId);
        ExecutionJobVertex stateless = graph.getJobVertex(statelessId);
        ExecutionVertex stateful1 = stateful.getTaskVertices()[0];
        ExecutionVertex stateful2 = stateful.getTaskVertices()[1];
        ExecutionVertex stateful3 = stateful.getTaskVertices()[2];
        ExecutionVertex stateless1 = stateless.getTaskVertices()[0];
        ExecutionVertex stateless2 = stateless.getTaskVertices()[1];
        Execution statefulExec1 = stateful1.getCurrentExecutionAttempt();
        Execution statefulExec2 = stateful2.getCurrentExecutionAttempt();
        Execution statefulExec3 = stateful3.getCurrentExecutionAttempt();
        Execution statelessExec1 = stateless1.getCurrentExecutionAttempt();
        Execution statelessExec2 = stateless2.getCurrentExecutionAttempt();
        ManuallyTriggeredScheduledExecutor manuallyTriggeredScheduledExecutor = new ManuallyTriggeredScheduledExecutor();
        CheckpointCoordinator coord = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setTimer(manuallyTriggeredScheduledExecutor).build();
        // create ourselves a checkpoint with state
        coord.triggerCheckpoint(false);
        manuallyTriggeredScheduledExecutor.triggerAll();
        PendingCheckpoint pending = coord.getPendingCheckpoints().values().iterator().next();
        final long checkpointId = pending.getCheckpointId();
        final TaskStateSnapshot subtaskStates = new TaskStateSnapshot();
        subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(statefulId), OperatorSubtaskState.builder().setManagedKeyedState(serializedKeyGroupStates).build());
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), statefulExec1.getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskStates), TASK_MANAGER_LOCATION_INFO);
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), statefulExec2.getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskStates), TASK_MANAGER_LOCATION_INFO);
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), statefulExec3.getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskStates), TASK_MANAGER_LOCATION_INFO);
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), statelessExec1.getAttemptId(), checkpointId), TASK_MANAGER_LOCATION_INFO);
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), statelessExec2.getAttemptId(), checkpointId), TASK_MANAGER_LOCATION_INFO);
        assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
        assertEquals(0, coord.getNumberOfPendingCheckpoints());
        // let the coordinator inject the state
        assertTrue(coord.restoreLatestCheckpointedStateToAll(new HashSet<>(Arrays.asList(stateful, stateless)), false));
        // verify that each stateful vertex got the state
        assertEquals(subtaskStates, statefulExec1.getTaskRestore().getTaskStateSnapshot());
        assertEquals(subtaskStates, statefulExec2.getTaskRestore().getTaskStateSnapshot());
        assertEquals(subtaskStates, statefulExec3.getTaskRestore().getTaskStateSnapshot());
        assertNull(statelessExec1.getTaskRestore());
        assertNull(statelessExec2.getTaskRestore());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 39 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class CheckpointStateRestoreTest method mockExecution.

private Execution mockExecution(ExecutionState state) {
    Execution mock = mock(Execution.class);
    when(mock.getAttemptId()).thenReturn(new ExecutionAttemptID());
    when(mock.getState()).thenReturn(state);
    return mock;
}

Example 40 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class TaskDeploymentDescriptorFactory method getConsumedPartitionShuffleDescriptor.

public static ShuffleDescriptor getConsumedPartitionShuffleDescriptor(IntermediateResultPartition consumedPartition, PartitionLocationConstraint partitionDeploymentConstraint) {
    Execution producer = consumedPartition.getProducer().getCurrentExecutionAttempt();
    ExecutionState producerState = producer.getState();
    Optional<ResultPartitionDeploymentDescriptor> consumedPartitionDescriptor = producer.getResultPartitionDeploymentDescriptor(consumedPartition.getPartitionId());
    ResultPartitionID consumedPartitionId = new ResultPartitionID(consumedPartition.getPartitionId(), producer.getAttemptId());
    return getConsumedPartitionShuffleDescriptor(consumedPartitionId, consumedPartition.getResultType(), consumedPartition.isConsumable(), producerState, partitionDeploymentConstraint, consumedPartitionDescriptor.orElse(null));
}