Search in sources :

Example 1 with ShuffleMaster

use of org.apache.flink.runtime.shuffle.ShuffleMaster in project flink by apache.

the class DefaultExecutionGraphFactory method createAndRestoreExecutionGraph.

@Override
public ExecutionGraph createAndRestoreExecutionGraph(JobGraph jobGraph, CompletedCheckpointStore completedCheckpointStore, CheckpointsCleaner checkpointsCleaner, CheckpointIDCounter checkpointIdCounter, TaskDeploymentDescriptorFactory.PartitionLocationConstraint partitionLocationConstraint, long initializationTimestamp, VertexAttemptNumberStore vertexAttemptNumberStore, VertexParallelismStore vertexParallelismStore, ExecutionStateUpdateListener executionStateUpdateListener, Logger log) throws Exception {
    ExecutionDeploymentListener executionDeploymentListener = new ExecutionDeploymentTrackerDeploymentListenerAdapter(executionDeploymentTracker);
    ExecutionStateUpdateListener combinedExecutionStateUpdateListener = (execution, previousState, newState) -> {
        executionStateUpdateListener.onStateUpdate(execution, previousState, newState);
        if (newState.isTerminal()) {
            executionDeploymentTracker.stopTrackingDeploymentOf(execution);
        }
    };
    final ExecutionGraph newExecutionGraph = DefaultExecutionGraphBuilder.buildGraph(jobGraph, configuration, futureExecutor, ioExecutor, userCodeClassLoader, completedCheckpointStore, checkpointsCleaner, checkpointIdCounter, rpcTimeout, blobWriter, log, shuffleMaster, jobMasterPartitionTracker, partitionLocationConstraint, executionDeploymentListener, combinedExecutionStateUpdateListener, initializationTimestamp, vertexAttemptNumberStore, vertexParallelismStore, checkpointStatsTrackerFactory, isDynamicGraph);
    final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();
    if (checkpointCoordinator != null) {
        // check whether we find a valid checkpoint
        if (!checkpointCoordinator.restoreInitialCheckpointIfPresent(new HashSet<>(newExecutionGraph.getAllVertices().values()))) {
            // check whether we can restore from a savepoint
            tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
        }
    }
    return newExecutionGraph;
}
Also used : ExecutionStateUpdateListener(org.apache.flink.runtime.executiongraph.ExecutionStateUpdateListener) ShuffleMaster(org.apache.flink.runtime.shuffle.ShuffleMaster) CheckpointCoordinator(org.apache.flink.runtime.checkpoint.CheckpointCoordinator) TaskDeploymentDescriptorFactory(org.apache.flink.runtime.deployment.TaskDeploymentDescriptorFactory) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) Supplier(java.util.function.Supplier) CachingSupplier(org.apache.flink.util.function.CachingSupplier) HashSet(java.util.HashSet) JobMasterPartitionTracker(org.apache.flink.runtime.io.network.partition.JobMasterPartitionTracker) CheckpointStatsTracker(org.apache.flink.runtime.checkpoint.CheckpointStatsTracker) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BlobWriter(org.apache.flink.runtime.blob.BlobWriter) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) JobManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup) CheckpointIDCounter(org.apache.flink.runtime.checkpoint.CheckpointIDCounter) Configuration(org.apache.flink.configuration.Configuration) CompletedCheckpointStore(org.apache.flink.runtime.checkpoint.CompletedCheckpointStore) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) ExecutionDeploymentTracker(org.apache.flink.runtime.jobmaster.ExecutionDeploymentTracker) DefaultExecutionGraphBuilder(org.apache.flink.runtime.executiongraph.DefaultExecutionGraphBuilder) ExecutionDeploymentTrackerDeploymentListenerAdapter(org.apache.flink.runtime.jobmaster.ExecutionDeploymentTrackerDeploymentListenerAdapter) VertexAttemptNumberStore(org.apache.flink.runtime.executiongraph.VertexAttemptNumberStore) ExecutionDeploymentListener(org.apache.flink.runtime.executiongraph.ExecutionDeploymentListener) WebOptions(org.apache.flink.configuration.WebOptions) SavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings) Time(org.apache.flink.api.common.time.Time) CheckpointCoordinator(org.apache.flink.runtime.checkpoint.CheckpointCoordinator) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) ExecutionDeploymentListener(org.apache.flink.runtime.executiongraph.ExecutionDeploymentListener) ExecutionDeploymentTrackerDeploymentListenerAdapter(org.apache.flink.runtime.jobmaster.ExecutionDeploymentTrackerDeploymentListenerAdapter) ExecutionStateUpdateListener(org.apache.flink.runtime.executiongraph.ExecutionStateUpdateListener) HashSet(java.util.HashSet)

Aggregations

HashSet (java.util.HashSet)1 Executor (java.util.concurrent.Executor)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 Supplier (java.util.function.Supplier)1 Time (org.apache.flink.api.common.time.Time)1 Configuration (org.apache.flink.configuration.Configuration)1 WebOptions (org.apache.flink.configuration.WebOptions)1 BlobWriter (org.apache.flink.runtime.blob.BlobWriter)1 CheckpointCoordinator (org.apache.flink.runtime.checkpoint.CheckpointCoordinator)1 CheckpointIDCounter (org.apache.flink.runtime.checkpoint.CheckpointIDCounter)1 CheckpointStatsTracker (org.apache.flink.runtime.checkpoint.CheckpointStatsTracker)1 CheckpointsCleaner (org.apache.flink.runtime.checkpoint.CheckpointsCleaner)1 CompletedCheckpointStore (org.apache.flink.runtime.checkpoint.CompletedCheckpointStore)1 TaskDeploymentDescriptorFactory (org.apache.flink.runtime.deployment.TaskDeploymentDescriptorFactory)1 DefaultExecutionGraphBuilder (org.apache.flink.runtime.executiongraph.DefaultExecutionGraphBuilder)1 ExecutionDeploymentListener (org.apache.flink.runtime.executiongraph.ExecutionDeploymentListener)1 ExecutionGraph (org.apache.flink.runtime.executiongraph.ExecutionGraph)1 ExecutionStateUpdateListener (org.apache.flink.runtime.executiongraph.ExecutionStateUpdateListener)1 VertexAttemptNumberStore (org.apache.flink.runtime.executiongraph.VertexAttemptNumberStore)1 JobMasterPartitionTracker (org.apache.flink.runtime.io.network.partition.JobMasterPartitionTracker)1