Search in sources :

Example 1 with UpdateSchedulerNgOnInternalFailuresListener

use of org.apache.flink.runtime.scheduler.UpdateSchedulerNgOnInternalFailuresListener in project flink by apache.

the class AdaptiveScheduler method tryToAssignSlots.

@Override
public CreatingExecutionGraph.AssignmentResult tryToAssignSlots(CreatingExecutionGraph.ExecutionGraphWithVertexParallelism executionGraphWithVertexParallelism) {
    final ExecutionGraph executionGraph = executionGraphWithVertexParallelism.getExecutionGraph();
    executionGraph.start(componentMainThreadExecutor);
    executionGraph.transitionToRunning();
    executionGraph.setInternalTaskFailuresListener(new UpdateSchedulerNgOnInternalFailuresListener(this));
    final VertexParallelism vertexParallelism = executionGraphWithVertexParallelism.getVertexParallelism();
    return slotAllocator.tryReserveResources(vertexParallelism).map(reservedSlots -> CreatingExecutionGraph.AssignmentResult.success(assignSlotsToExecutionGraph(executionGraph, reservedSlots))).orElseGet(CreatingExecutionGraph.AssignmentResult::notPossible);
}
Also used : UpdateSchedulerNgOnInternalFailuresListener(org.apache.flink.runtime.scheduler.UpdateSchedulerNgOnInternalFailuresListener) ExecutionGraphFactory(org.apache.flink.runtime.scheduler.ExecutionGraphFactory) DeclarativeSlotPool(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPool) TaskNotRunningException(org.apache.flink.runtime.operators.coordination.TaskNotRunningException) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) PhysicalSlot(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlot) Duration(java.time.Duration) FunctionWithException(org.apache.flink.util.function.FunctionWithException) ExecutionGraphHandler(org.apache.flink.runtime.scheduler.ExecutionGraphHandler) JobManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup) ReservedSlots(org.apache.flink.runtime.scheduler.adaptive.allocator.ReservedSlots) MetricOptions(org.apache.flink.configuration.MetricOptions) JobManagerOptions(org.apache.flink.configuration.JobManagerOptions) ExecutionFailureHandler(org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler) JobStatusStore(org.apache.flink.runtime.scheduler.JobStatusStore) VertexParallelismStore(org.apache.flink.runtime.scheduler.VertexParallelismStore) CheckpointScheduling(org.apache.flink.runtime.checkpoint.CheckpointScheduling) RootExceptionHistoryEntry(org.apache.flink.runtime.scheduler.exceptionhistory.RootExceptionHistoryEntry) FlinkException(org.apache.flink.util.FlinkException) ComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor) JobEdge(org.apache.flink.runtime.jobgraph.JobEdge) CoordinationResponse(org.apache.flink.runtime.operators.coordination.CoordinationResponse) ExceptionHistoryEntry(org.apache.flink.runtime.scheduler.exceptionhistory.ExceptionHistoryEntry) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) ResourceCounter(org.apache.flink.runtime.util.ResourceCounter) JobStatus(org.apache.flink.api.common.JobStatus) CheckpointFailureReason(org.apache.flink.runtime.checkpoint.CheckpointFailureReason) ArrayList(java.util.ArrayList) SchedulerNG(org.apache.flink.runtime.scheduler.SchedulerNG) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) PartitionProducerDisposedException(org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException) JobExecutionException(org.apache.flink.runtime.client.JobExecutionException) Nullable(javax.annotation.Nullable) DefaultVertexParallelismStore(org.apache.flink.runtime.scheduler.DefaultVertexParallelismStore) KvStateLocation(org.apache.flink.runtime.query.KvStateLocation) Executor(java.util.concurrent.Executor) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) VertexParallelism(org.apache.flink.runtime.scheduler.adaptive.allocator.VertexParallelism) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) IOException(java.io.IOException) OperatorCoordinatorHandler(org.apache.flink.runtime.scheduler.OperatorCoordinatorHandler) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) RestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy) ReactiveScaleUpController(org.apache.flink.runtime.scheduler.adaptive.scalingpolicy.ReactiveScaleUpController) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) JobID(org.apache.flink.api.common.JobID) SlotInfo(org.apache.flink.runtime.jobmaster.SlotInfo) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex) KvStateID(org.apache.flink.queryablestate.KvStateID) ScheduledFuture(java.util.concurrent.ScheduledFuture) ScaleUpController(org.apache.flink.runtime.scheduler.adaptive.scalingpolicy.ScaleUpController) TaskDeploymentDescriptorFactory(org.apache.flink.runtime.deployment.TaskDeploymentDescriptorFactory) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) LoggerFactory(org.slf4j.LoggerFactory) ExceptionUtils(org.apache.flink.util.ExceptionUtils) UpdateSchedulerNgOnInternalFailuresListener(org.apache.flink.runtime.scheduler.UpdateSchedulerNgOnInternalFailuresListener) MutableVertexAttemptNumberStore(org.apache.flink.runtime.executiongraph.MutableVertexAttemptNumberStore) BoundedFIFOQueue(org.apache.flink.runtime.util.BoundedFIFOQueue) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) UnknownKvStateLocation(org.apache.flink.runtime.query.UnknownKvStateLocation) SerializedInputSplit(org.apache.flink.runtime.jobmaster.SerializedInputSplit) JobDetails(org.apache.flink.runtime.messages.webmonitor.JobDetails) CheckpointIDCounter(org.apache.flink.runtime.checkpoint.CheckpointIDCounter) Collection(java.util.Collection) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) Preconditions(org.apache.flink.util.Preconditions) InetSocketAddress(java.net.InetSocketAddress) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) List(java.util.List) CoordinationRequest(org.apache.flink.runtime.operators.coordination.CoordinationRequest) FlinkJobNotFoundException(org.apache.flink.runtime.messages.FlinkJobNotFoundException) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) SchedulerExecutionMode(org.apache.flink.configuration.SchedulerExecutionMode) NoResourceAvailableException(org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException) SchedulerBase(org.apache.flink.runtime.scheduler.SchedulerBase) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) SchedulerUtils(org.apache.flink.runtime.scheduler.SchedulerUtils) SlotAllocator(org.apache.flink.runtime.scheduler.adaptive.allocator.SlotAllocator) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) JobType(org.apache.flink.runtime.jobgraph.JobType) VertexParallelismInformation(org.apache.flink.runtime.scheduler.VertexParallelismInformation) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) JobException(org.apache.flink.runtime.JobException) DeploymentStateTimeMetrics(org.apache.flink.runtime.scheduler.metrics.DeploymentStateTimeMetrics) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) DefaultVertexParallelismInfo(org.apache.flink.runtime.scheduler.DefaultVertexParallelismInfo) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Nonnull(javax.annotation.Nonnull) ArchivedExecutionGraph(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Configuration(org.apache.flink.configuration.Configuration) CompletedCheckpointStore(org.apache.flink.runtime.checkpoint.CompletedCheckpointStore) LogicalSlot(org.apache.flink.runtime.jobmaster.LogicalSlot) AccumulatorSnapshot(org.apache.flink.runtime.accumulators.AccumulatorSnapshot) JobStatusListener(org.apache.flink.runtime.executiongraph.JobStatusListener) DeclineCheckpoint(org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint) ExecutionGraphInfo(org.apache.flink.runtime.scheduler.ExecutionGraphInfo) CheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory) TimeUnit(java.util.concurrent.TimeUnit) WebOptions(org.apache.flink.configuration.WebOptions) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) DefaultVertexAttemptNumberStore(org.apache.flink.runtime.executiongraph.DefaultVertexAttemptNumberStore) Collections(java.util.Collections) OperatorEvent(org.apache.flink.runtime.operators.coordination.OperatorEvent) VertexParallelism(org.apache.flink.runtime.scheduler.adaptive.allocator.VertexParallelism) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) ArchivedExecutionGraph(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph)

Aggregations

IOException (java.io.IOException)1 InetSocketAddress (java.net.InetSocketAddress)1 Duration (java.time.Duration)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Optional (java.util.Optional)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 Executor (java.util.concurrent.Executor)1 ScheduledFuture (java.util.concurrent.ScheduledFuture)1 TimeUnit (java.util.concurrent.TimeUnit)1 Function (java.util.function.Function)1 Nonnull (javax.annotation.Nonnull)1 Nullable (javax.annotation.Nullable)1 VisibleForTesting (org.apache.flink.annotation.VisibleForTesting)1 JobID (org.apache.flink.api.common.JobID)1 JobStatus (org.apache.flink.api.common.JobStatus)1 Configuration (org.apache.flink.configuration.Configuration)1