Examples with SlotSelectionStrategy org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy used on opensource projects

Example 1 with SlotSelectionStrategy

use of org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy in project flink by apache.

the class AdaptiveBatchSchedulerFactory method createInstance.

@Override
public SchedulerNG createInstance(Logger log, JobGraph jobGraph, Executor ioExecutor, Configuration jobMasterConfiguration, SlotPoolService slotPoolService, ScheduledExecutorService futureExecutor, ClassLoader userCodeLoader, CheckpointRecoveryFactory checkpointRecoveryFactory, Time rpcTimeout, BlobWriter blobWriter, JobManagerJobMetricGroup jobManagerJobMetricGroup, Time slotRequestTimeout, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionDeploymentTracker executionDeploymentTracker, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, FatalErrorHandler fatalErrorHandler, JobStatusListener jobStatusListener) throws Exception {
    checkState(jobGraph.getJobType() == JobType.BATCH, "Adaptive batch scheduler only supports batch jobs");
    checkAllExchangesBlocking(jobGraph);
    final SlotPool slotPool = slotPoolService.castInto(SlotPool.class).orElseThrow(() -> new IllegalStateException("The DefaultScheduler requires a SlotPool."));
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(JobType.BATCH, jobMasterConfiguration);
    final PhysicalSlotRequestBulkChecker bulkChecker = PhysicalSlotRequestBulkCheckerImpl.createFromSlotPool(slotPool, SystemClock.getInstance());
    final PhysicalSlotProvider physicalSlotProvider = new PhysicalSlotProviderImpl(slotSelectionStrategy, slotPool);
    final ExecutionSlotAllocatorFactory allocatorFactory = new SlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider, false, bulkChecker, slotRequestTimeout);
    final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader.createRestartBackoffTimeStrategyFactory(jobGraph.getSerializedExecutionConfig().deserializeValue(userCodeLoader).getRestartStrategy(), jobMasterConfiguration, jobGraph.isCheckpointingEnabled()).create();
    log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());
    final ExecutionGraphFactory executionGraphFactory = new DefaultExecutionGraphFactory(jobMasterConfiguration, userCodeLoader, executionDeploymentTracker, futureExecutor, ioExecutor, rpcTimeout, jobManagerJobMetricGroup, blobWriter, shuffleMaster, partitionTracker, true);
    return new AdaptiveBatchScheduler(log, jobGraph, ioExecutor, jobMasterConfiguration, bulkChecker::start, new ScheduledExecutorServiceAdapter(futureExecutor), userCodeLoader, new CheckpointsCleaner(), checkpointRecoveryFactory, jobManagerJobMetricGroup, new VertexwiseSchedulingStrategy.Factory(), FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration), restartBackoffTimeStrategy, new DefaultExecutionVertexOperations(), new ExecutionVertexVersioner(), allocatorFactory, initializationTimestamp, mainThreadExecutor, jobStatusListener, executionGraphFactory, shuffleMaster, rpcTimeout, DefaultVertexParallelismDecider.from(jobMasterConfiguration), jobMasterConfiguration.getInteger(JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MAX_PARALLELISM));
}

Example 2 with SlotSelectionStrategy

use of org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy in project flink by apache.

the class SlotSelectionStrategyUtils method selectSlotSelectionStrategy.

public static SlotSelectionStrategy selectSlotSelectionStrategy(final JobType jobType, final Configuration configuration) {
    final boolean evenlySpreadOutSlots = configuration.getBoolean(ClusterOptions.EVENLY_SPREAD_OUT_SLOTS_STRATEGY);
    final SlotSelectionStrategy locationPreferenceSlotSelectionStrategy;
    locationPreferenceSlotSelectionStrategy = evenlySpreadOutSlots ? LocationPreferenceSlotSelectionStrategy.createEvenlySpreadOut() : LocationPreferenceSlotSelectionStrategy.createDefault();
    final boolean isLocalRecoveryEnabled = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);
    if (isLocalRecoveryEnabled) {
        if (jobType == JobType.STREAMING) {
            return PreviousAllocationSlotSelectionStrategy.create(locationPreferenceSlotSelectionStrategy);
        } else {
            LOG.warn("Batch job does not support local recovery. Falling back to use " + locationPreferenceSlotSelectionStrategy.getClass());
            return locationPreferenceSlotSelectionStrategy;
        }
    } else {
        return locationPreferenceSlotSelectionStrategy;
    }
}

Example 3 with SlotSelectionStrategy

use of org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy in project flink by apache.

the class DefaultSchedulerComponents method createPipelinedRegionSchedulerComponents.

private static DefaultSchedulerComponents createPipelinedRegionSchedulerComponents(final JobType jobType, final Configuration jobMasterConfiguration, final SlotPool slotPool, final Time slotRequestTimeout) {
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(jobType, jobMasterConfiguration);
    final PhysicalSlotRequestBulkChecker bulkChecker = PhysicalSlotRequestBulkCheckerImpl.createFromSlotPool(slotPool, SystemClock.getInstance());
    final PhysicalSlotProvider physicalSlotProvider = new PhysicalSlotProviderImpl(slotSelectionStrategy, slotPool);
    final ExecutionSlotAllocatorFactory allocatorFactory = new SlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider, jobType == JobType.STREAMING, bulkChecker, slotRequestTimeout);
    return new DefaultSchedulerComponents(new PipelinedRegionSchedulingStrategy.Factory(), bulkChecker::start, allocatorFactory);
}

Example 4 with SlotSelectionStrategy

use of org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy in project flink by apache.

the class SlotSelectionStrategyUtilsTest method testCreatePreviousAllocationSlotSelectionStrategyForLocalRecoveryStreamingJob.

@Test
public void testCreatePreviousAllocationSlotSelectionStrategyForLocalRecoveryStreamingJob() {
    final Configuration configuration = new Configuration();
    configuration.set(CheckpointingOptions.LOCAL_RECOVERY, true);
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(JobType.STREAMING, configuration);
    assertThat(slotSelectionStrategy, instanceOf(PreviousAllocationSlotSelectionStrategy.class));
}

Example 5 with SlotSelectionStrategy

use of org.apache.flink.runtime.jobmaster.slotpool.SlotSelectionStrategy in project flink by apache.

the class SlotSelectionStrategyUtilsTest method testCreateLocationPreferenceSlotSelectionStrategyForLocalRecoveryBatchJob.

@Test
public void testCreateLocationPreferenceSlotSelectionStrategyForLocalRecoveryBatchJob() {
    final Configuration configuration = new Configuration();
    configuration.set(CheckpointingOptions.LOCAL_RECOVERY, true);
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(JobType.BATCH, configuration);
    assertThat(slotSelectionStrategy, instanceOf(LocationPreferenceSlotSelectionStrategy.class));
}