Search in sources :

Example 1 with Vertex

use of org.apache.flink.runtime.topology.Vertex in project flink by apache.

the class DefaultScheduler method deployOrHandleError.

private BiFunction<Object, Throwable, Void> deployOrHandleError(final DeploymentHandle deploymentHandle) {
    final ExecutionVertexVersion requiredVertexVersion = deploymentHandle.getRequiredVertexVersion();
    final ExecutionVertexID executionVertexId = requiredVertexVersion.getExecutionVertexId();
    return (ignored, throwable) -> {
        if (executionVertexVersioner.isModified(requiredVertexVersion)) {
            log.debug("Refusing to deploy execution vertex {} because this deployment was " + "superseded by another deployment", executionVertexId);
            return null;
        }
        if (throwable == null) {
            deployTaskSafe(executionVertexId);
        } else {
            handleTaskDeploymentFailure(executionVertexId, throwable);
        }
        return null;
    };
}
Also used : ShuffleMaster(org.apache.flink.runtime.shuffle.ShuffleMaster) TaskManagerLocation(org.apache.flink.runtime.taskmanager.TaskManagerLocation) BiFunction(java.util.function.BiFunction) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) Vertex(org.apache.flink.runtime.topology.Vertex) Map(java.util.Map) SchedulingTopology(org.apache.flink.runtime.scheduler.strategy.SchedulingTopology) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) CoLocationGroup(org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup) SchedulingStrategyFactory(org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory) ScheduledExecutor(org.apache.flink.util.concurrent.ScheduledExecutor) JobManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup) Collection(java.util.Collection) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) ExecutionVertexID(org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID) Collectors(java.util.stream.Collectors) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) List(java.util.List) FailoverStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.FailoverStrategy) Optional(java.util.Optional) ExecutionFailureHandler(org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler) Time(org.apache.flink.api.common.time.Time) AllocationID(org.apache.flink.runtime.clusterframework.types.AllocationID) NoResourceAvailableException(org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) ComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor) SlotSharingGroup(org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) Function(java.util.function.Function) ArrayList(java.util.ArrayList) FailureHandlingResult(org.apache.flink.runtime.executiongraph.failover.flip1.FailureHandlingResult) HashSet(java.util.HashSet) OperatorCoordinatorHolder(org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) SchedulingStrategy(org.apache.flink.runtime.scheduler.strategy.SchedulingStrategy) Nullable(javax.annotation.Nullable) Preconditions.checkState(org.apache.flink.util.Preconditions.checkState) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) Configuration(org.apache.flink.configuration.Configuration) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) LogicalSlot(org.apache.flink.runtime.jobmaster.LogicalSlot) IterableUtils(org.apache.flink.util.IterableUtils) JobStatusListener(org.apache.flink.runtime.executiongraph.JobStatusListener) CheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory) RestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) FailureHandlingResultSnapshot(org.apache.flink.runtime.scheduler.exceptionhistory.FailureHandlingResultSnapshot) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex) ExecutionVertexID(org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID)

Example 2 with Vertex

use of org.apache.flink.runtime.topology.Vertex in project flink by apache.

the class DefaultScheduler method assignResource.

private BiFunction<LogicalSlot, Throwable, LogicalSlot> assignResource(final DeploymentHandle deploymentHandle) {
    final ExecutionVertexVersion requiredVertexVersion = deploymentHandle.getRequiredVertexVersion();
    final ExecutionVertexID executionVertexId = deploymentHandle.getExecutionVertexId();
    return (logicalSlot, throwable) -> {
        if (executionVertexVersioner.isModified(requiredVertexVersion)) {
            if (throwable == null) {
                log.debug("Refusing to assign slot to execution vertex {} because this deployment was " + "superseded by another deployment", executionVertexId);
                releaseSlotIfPresent(logicalSlot);
            }
            return null;
        }
        // a task which is about to cancel in #restartTasksWithDelay(...)
        if (throwable != null) {
            throw new CompletionException(maybeWrapWithNoResourceAvailableException(throwable));
        }
        final ExecutionVertex executionVertex = getExecutionVertex(executionVertexId);
        executionVertex.tryAssignResource(logicalSlot);
        startReserveAllocation(executionVertexId, logicalSlot.getAllocationId());
        return logicalSlot;
    };
}
Also used : ShuffleMaster(org.apache.flink.runtime.shuffle.ShuffleMaster) TaskManagerLocation(org.apache.flink.runtime.taskmanager.TaskManagerLocation) BiFunction(java.util.function.BiFunction) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) Vertex(org.apache.flink.runtime.topology.Vertex) Map(java.util.Map) SchedulingTopology(org.apache.flink.runtime.scheduler.strategy.SchedulingTopology) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) CoLocationGroup(org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup) SchedulingStrategyFactory(org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory) ScheduledExecutor(org.apache.flink.util.concurrent.ScheduledExecutor) JobManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup) Collection(java.util.Collection) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) ExecutionVertexID(org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID) Collectors(java.util.stream.Collectors) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) List(java.util.List) FailoverStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.FailoverStrategy) Optional(java.util.Optional) ExecutionFailureHandler(org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler) Time(org.apache.flink.api.common.time.Time) AllocationID(org.apache.flink.runtime.clusterframework.types.AllocationID) NoResourceAvailableException(org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) ComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor) SlotSharingGroup(org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) Function(java.util.function.Function) ArrayList(java.util.ArrayList) FailureHandlingResult(org.apache.flink.runtime.executiongraph.failover.flip1.FailureHandlingResult) HashSet(java.util.HashSet) OperatorCoordinatorHolder(org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) SchedulingStrategy(org.apache.flink.runtime.scheduler.strategy.SchedulingStrategy) Nullable(javax.annotation.Nullable) Preconditions.checkState(org.apache.flink.util.Preconditions.checkState) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Logger(org.slf4j.Logger) Executor(java.util.concurrent.Executor) Configuration(org.apache.flink.configuration.Configuration) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) LogicalSlot(org.apache.flink.runtime.jobmaster.LogicalSlot) IterableUtils(org.apache.flink.util.IterableUtils) JobStatusListener(org.apache.flink.runtime.executiongraph.JobStatusListener) CheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory) RestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) FailureHandlingResultSnapshot(org.apache.flink.runtime.scheduler.exceptionhistory.FailureHandlingResultSnapshot) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex) ExecutionVertexID(org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID) CompletionException(java.util.concurrent.CompletionException) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex)

Aggregations

ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 Optional (java.util.Optional)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 CompletionException (java.util.concurrent.CompletionException)2 Executor (java.util.concurrent.Executor)2 TimeUnit (java.util.concurrent.TimeUnit)2 TimeoutException (java.util.concurrent.TimeoutException)2 BiFunction (java.util.function.BiFunction)2 Consumer (java.util.function.Consumer)2 Function (java.util.function.Function)2 Collectors (java.util.stream.Collectors)2 Nullable (javax.annotation.Nullable)2 JobStatus (org.apache.flink.api.common.JobStatus)2 Time (org.apache.flink.api.common.time.Time)2