Search in sources :

Example 1 with ExecutionGraphFactory

use of org.apache.flink.runtime.scheduler.ExecutionGraphFactory in project flink by apache.

the class AdaptiveBatchSchedulerFactory method createInstance.

@Override
public SchedulerNG createInstance(Logger log, JobGraph jobGraph, Executor ioExecutor, Configuration jobMasterConfiguration, SlotPoolService slotPoolService, ScheduledExecutorService futureExecutor, ClassLoader userCodeLoader, CheckpointRecoveryFactory checkpointRecoveryFactory, Time rpcTimeout, BlobWriter blobWriter, JobManagerJobMetricGroup jobManagerJobMetricGroup, Time slotRequestTimeout, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionDeploymentTracker executionDeploymentTracker, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, FatalErrorHandler fatalErrorHandler, JobStatusListener jobStatusListener) throws Exception {
    checkState(jobGraph.getJobType() == JobType.BATCH, "Adaptive batch scheduler only supports batch jobs");
    checkAllExchangesBlocking(jobGraph);
    final SlotPool slotPool = slotPoolService.castInto(SlotPool.class).orElseThrow(() -> new IllegalStateException("The DefaultScheduler requires a SlotPool."));
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(JobType.BATCH, jobMasterConfiguration);
    final PhysicalSlotRequestBulkChecker bulkChecker = PhysicalSlotRequestBulkCheckerImpl.createFromSlotPool(slotPool, SystemClock.getInstance());
    final PhysicalSlotProvider physicalSlotProvider = new PhysicalSlotProviderImpl(slotSelectionStrategy, slotPool);
    final ExecutionSlotAllocatorFactory allocatorFactory = new SlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider, false, bulkChecker, slotRequestTimeout);
    final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader.createRestartBackoffTimeStrategyFactory(jobGraph.getSerializedExecutionConfig().deserializeValue(userCodeLoader).getRestartStrategy(), jobMasterConfiguration, jobGraph.isCheckpointingEnabled()).create();
    log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());
    final ExecutionGraphFactory executionGraphFactory = new DefaultExecutionGraphFactory(jobMasterConfiguration, userCodeLoader, executionDeploymentTracker, futureExecutor, ioExecutor, rpcTimeout, jobManagerJobMetricGroup, blobWriter, shuffleMaster, partitionTracker, true);
    return new AdaptiveBatchScheduler(log, jobGraph, ioExecutor, jobMasterConfiguration, bulkChecker::start, new ScheduledExecutorServiceAdapter(futureExecutor), userCodeLoader, new CheckpointsCleaner(), checkpointRecoveryFactory, jobManagerJobMetricGroup, new VertexwiseSchedulingStrategy.Factory(), FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration), restartBackoffTimeStrategy, new DefaultExecutionVertexOperations(), new ExecutionVertexVersioner(), allocatorFactory, initializationTimestamp, mainThreadExecutor, jobStatusListener, executionGraphFactory, shuffleMaster, rpcTimeout, DefaultVertexParallelismDecider.from(jobMasterConfiguration), jobMasterConfiguration.getInteger(JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MAX_PARALLELISM));
}
Also used : DefaultExecutionVertexOperations(org.apache.flink.runtime.scheduler.DefaultExecutionVertexOperations) SlotSharingExecutionSlotAllocatorFactory(org.apache.flink.runtime.scheduler.SlotSharingExecutionSlotAllocatorFactory) SlotSelectionStrategy(org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy) VertexwiseSchedulingStrategy(org.apache.flink.runtime.scheduler.strategy.VertexwiseSchedulingStrategy) SlotPool(org.apache.flink.runtime.jobmaster.slotpool.SlotPool) PhysicalSlotRequestBulkChecker(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotRequestBulkChecker) PhysicalSlotProviderImpl(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProviderImpl) SlotSharingExecutionSlotAllocatorFactory(org.apache.flink.runtime.scheduler.SlotSharingExecutionSlotAllocatorFactory) ExecutionSlotAllocatorFactory(org.apache.flink.runtime.scheduler.ExecutionSlotAllocatorFactory) ScheduledExecutorServiceAdapter(org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter) RestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) DefaultExecutionGraphFactory(org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory) ExecutionVertexVersioner(org.apache.flink.runtime.scheduler.ExecutionVertexVersioner) ExecutionGraphFactory(org.apache.flink.runtime.scheduler.ExecutionGraphFactory) DefaultExecutionGraphFactory(org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory) PhysicalSlotProvider(org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProvider)

Example 2 with ExecutionGraphFactory

use of org.apache.flink.runtime.scheduler.ExecutionGraphFactory in project flink by apache.

the class AdaptiveSchedulerFactory method createInstance.

@Override
public SchedulerNG createInstance(Logger log, JobGraph jobGraph, Executor ioExecutor, Configuration jobMasterConfiguration, SlotPoolService slotPoolService, ScheduledExecutorService futureExecutor, ClassLoader userCodeLoader, CheckpointRecoveryFactory checkpointRecoveryFactory, Time rpcTimeout, BlobWriter blobWriter, JobManagerJobMetricGroup jobManagerJobMetricGroup, Time slotRequestTimeout, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionDeploymentTracker executionDeploymentTracker, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, FatalErrorHandler fatalErrorHandler, JobStatusListener jobStatusListener) throws Exception {
    final DeclarativeSlotPool declarativeSlotPool = slotPoolService.castInto(DeclarativeSlotPool.class).orElseThrow(() -> new IllegalStateException("The AdaptiveScheduler requires a DeclarativeSlotPool."));
    final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader.createRestartBackoffTimeStrategyFactory(jobGraph.getSerializedExecutionConfig().deserializeValue(userCodeLoader).getRestartStrategy(), jobMasterConfiguration, jobGraph.isCheckpointingEnabled()).create();
    log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());
    final SlotSharingSlotAllocator slotAllocator = createSlotSharingSlotAllocator(declarativeSlotPool);
    final ExecutionGraphFactory executionGraphFactory = new DefaultExecutionGraphFactory(jobMasterConfiguration, userCodeLoader, executionDeploymentTracker, futureExecutor, ioExecutor, rpcTimeout, jobManagerJobMetricGroup, blobWriter, shuffleMaster, partitionTracker);
    return new AdaptiveScheduler(jobGraph, jobMasterConfiguration, declarativeSlotPool, slotAllocator, ioExecutor, userCodeLoader, new CheckpointsCleaner(), checkpointRecoveryFactory, initialResourceAllocationTimeout, resourceStabilizationTimeout, jobManagerJobMetricGroup, restartBackoffTimeStrategy, initializationTimestamp, mainThreadExecutor, fatalErrorHandler, jobStatusListener, executionGraphFactory);
}
Also used : DeclarativeSlotPool(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPool) RestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) DefaultExecutionGraphFactory(org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory) SlotSharingSlotAllocator(org.apache.flink.runtime.scheduler.adaptive.allocator.SlotSharingSlotAllocator) ExecutionGraphFactory(org.apache.flink.runtime.scheduler.ExecutionGraphFactory) DefaultExecutionGraphFactory(org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory)

Aggregations

CheckpointsCleaner (org.apache.flink.runtime.checkpoint.CheckpointsCleaner)2 RestartBackoffTimeStrategy (org.apache.flink.runtime.executiongraph.failover.flip1.RestartBackoffTimeStrategy)2 DefaultExecutionGraphFactory (org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory)2 ExecutionGraphFactory (org.apache.flink.runtime.scheduler.ExecutionGraphFactory)2 DeclarativeSlotPool (org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPool)1 PhysicalSlotProvider (org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProvider)1 PhysicalSlotProviderImpl (org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotProviderImpl)1 PhysicalSlotRequestBulkChecker (org.apache.flink.runtime.jobmaster.slotpool.PhysicalSlotRequestBulkChecker)1 SlotPool (org.apache.flink.runtime.jobmaster.slotpool.SlotPool)1 SlotSelectionStrategy (org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy)1 DefaultExecutionVertexOperations (org.apache.flink.runtime.scheduler.DefaultExecutionVertexOperations)1 ExecutionSlotAllocatorFactory (org.apache.flink.runtime.scheduler.ExecutionSlotAllocatorFactory)1 ExecutionVertexVersioner (org.apache.flink.runtime.scheduler.ExecutionVertexVersioner)1 SlotSharingExecutionSlotAllocatorFactory (org.apache.flink.runtime.scheduler.SlotSharingExecutionSlotAllocatorFactory)1 SlotSharingSlotAllocator (org.apache.flink.runtime.scheduler.adaptive.allocator.SlotSharingSlotAllocator)1 VertexwiseSchedulingStrategy (org.apache.flink.runtime.scheduler.strategy.VertexwiseSchedulingStrategy)1 ScheduledExecutorServiceAdapter (org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter)1