Search in sources :

Example 1 with DeclarativeSlotPoolBridgeServiceFactory

use of org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory in project flink by apache.

the class DefaultSlotPoolServiceSchedulerFactory method fromConfiguration.

public static DefaultSlotPoolServiceSchedulerFactory fromConfiguration(Configuration configuration, JobType jobType) {
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT));
    final Time batchSlotTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
    final SlotPoolServiceFactory slotPoolServiceFactory;
    final SchedulerNGFactory schedulerNGFactory;
    JobManagerOptions.SchedulerType schedulerType = ClusterOptions.getSchedulerType(configuration);
    if (schedulerType == JobManagerOptions.SchedulerType.Adaptive && jobType == JobType.BATCH) {
        LOG.info("Adaptive Scheduler configured, but Batch job detected. Changing scheduler type to NG / DefaultScheduler.");
        // overwrite
        schedulerType = JobManagerOptions.SchedulerType.Ng;
    }
    switch(schedulerType) {
        case Ng:
            schedulerNGFactory = new DefaultSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        case Adaptive:
            schedulerNGFactory = getAdaptiveSchedulerFactoryFromConfiguration(configuration);
            slotPoolServiceFactory = new DeclarativeSlotPoolServiceFactory(SystemClock.getInstance(), slotIdleTimeout, rpcTimeout);
            break;
        case AdaptiveBatch:
            schedulerNGFactory = new AdaptiveBatchSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        default:
            throw new IllegalArgumentException(String.format("Illegal value [%s] for config option [%s]", schedulerType, JobManagerOptions.SCHEDULER.key()));
    }
    return new DefaultSlotPoolServiceSchedulerFactory(slotPoolServiceFactory, schedulerNGFactory);
}
Also used : SchedulerNGFactory(org.apache.flink.runtime.scheduler.SchedulerNGFactory) AdaptiveBatchSchedulerFactory(org.apache.flink.runtime.scheduler.adaptivebatch.AdaptiveBatchSchedulerFactory) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) DeclarativeSlotPoolBridgeServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory) DefaultSchedulerFactory(org.apache.flink.runtime.scheduler.DefaultSchedulerFactory) Time(org.apache.flink.api.common.time.Time) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) SlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.SlotPoolServiceFactory) JobManagerOptions(org.apache.flink.configuration.JobManagerOptions)

Example 2 with DeclarativeSlotPoolBridgeServiceFactory

use of org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory in project flink-mirror by flink-ci.

the class DefaultSlotPoolServiceSchedulerFactory method fromConfiguration.

public static DefaultSlotPoolServiceSchedulerFactory fromConfiguration(Configuration configuration, JobType jobType) {
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT));
    final Time batchSlotTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
    final SlotPoolServiceFactory slotPoolServiceFactory;
    final SchedulerNGFactory schedulerNGFactory;
    JobManagerOptions.SchedulerType schedulerType = ClusterOptions.getSchedulerType(configuration);
    if (schedulerType == JobManagerOptions.SchedulerType.Adaptive && jobType == JobType.BATCH) {
        LOG.info("Adaptive Scheduler configured, but Batch job detected. Changing scheduler type to NG / DefaultScheduler.");
        // overwrite
        schedulerType = JobManagerOptions.SchedulerType.Ng;
    }
    switch(schedulerType) {
        case Ng:
            schedulerNGFactory = new DefaultSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        case Adaptive:
            schedulerNGFactory = getAdaptiveSchedulerFactoryFromConfiguration(configuration);
            slotPoolServiceFactory = new DeclarativeSlotPoolServiceFactory(SystemClock.getInstance(), slotIdleTimeout, rpcTimeout);
            break;
        case AdaptiveBatch:
            schedulerNGFactory = new AdaptiveBatchSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        default:
            throw new IllegalArgumentException(String.format("Illegal value [%s] for config option [%s]", schedulerType, JobManagerOptions.SCHEDULER.key()));
    }
    return new DefaultSlotPoolServiceSchedulerFactory(slotPoolServiceFactory, schedulerNGFactory);
}
Also used : SchedulerNGFactory(org.apache.flink.runtime.scheduler.SchedulerNGFactory) AdaptiveBatchSchedulerFactory(org.apache.flink.runtime.scheduler.adaptivebatch.AdaptiveBatchSchedulerFactory) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) DeclarativeSlotPoolBridgeServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory) DefaultSchedulerFactory(org.apache.flink.runtime.scheduler.DefaultSchedulerFactory) Time(org.apache.flink.api.common.time.Time) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) SlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.SlotPoolServiceFactory) JobManagerOptions(org.apache.flink.configuration.JobManagerOptions)

Example 3 with DeclarativeSlotPoolBridgeServiceFactory

use of org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory in project flink by splunk.

the class DefaultSlotPoolServiceSchedulerFactory method fromConfiguration.

public static DefaultSlotPoolServiceSchedulerFactory fromConfiguration(Configuration configuration, JobType jobType) {
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT));
    final Time batchSlotTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
    final SlotPoolServiceFactory slotPoolServiceFactory;
    final SchedulerNGFactory schedulerNGFactory;
    JobManagerOptions.SchedulerType schedulerType = ClusterOptions.getSchedulerType(configuration);
    if (schedulerType == JobManagerOptions.SchedulerType.Adaptive && jobType == JobType.BATCH) {
        LOG.info("Adaptive Scheduler configured, but Batch job detected. Changing scheduler type to NG / DefaultScheduler.");
        // overwrite
        schedulerType = JobManagerOptions.SchedulerType.Ng;
    }
    switch(schedulerType) {
        case Ng:
            schedulerNGFactory = new DefaultSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        case Adaptive:
            schedulerNGFactory = getAdaptiveSchedulerFactoryFromConfiguration(configuration);
            slotPoolServiceFactory = new DeclarativeSlotPoolServiceFactory(SystemClock.getInstance(), slotIdleTimeout, rpcTimeout);
            break;
        case AdaptiveBatch:
            schedulerNGFactory = new AdaptiveBatchSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        default:
            throw new IllegalArgumentException(String.format("Illegal value [%s] for config option [%s]", schedulerType, JobManagerOptions.SCHEDULER.key()));
    }
    return new DefaultSlotPoolServiceSchedulerFactory(slotPoolServiceFactory, schedulerNGFactory);
}
Also used : SchedulerNGFactory(org.apache.flink.runtime.scheduler.SchedulerNGFactory) AdaptiveBatchSchedulerFactory(org.apache.flink.runtime.scheduler.adaptivebatch.AdaptiveBatchSchedulerFactory) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) DeclarativeSlotPoolBridgeServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory) DefaultSchedulerFactory(org.apache.flink.runtime.scheduler.DefaultSchedulerFactory) Time(org.apache.flink.api.common.time.Time) DeclarativeSlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory) SlotPoolServiceFactory(org.apache.flink.runtime.jobmaster.slotpool.SlotPoolServiceFactory) JobManagerOptions(org.apache.flink.configuration.JobManagerOptions)

Aggregations

Time (org.apache.flink.api.common.time.Time)3 JobManagerOptions (org.apache.flink.configuration.JobManagerOptions)3 DeclarativeSlotPoolBridgeServiceFactory (org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridgeServiceFactory)3 DeclarativeSlotPoolServiceFactory (org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolServiceFactory)3 SlotPoolServiceFactory (org.apache.flink.runtime.jobmaster.slotpool.SlotPoolServiceFactory)3 DefaultSchedulerFactory (org.apache.flink.runtime.scheduler.DefaultSchedulerFactory)3 SchedulerNGFactory (org.apache.flink.runtime.scheduler.SchedulerNGFactory)3 AdaptiveBatchSchedulerFactory (org.apache.flink.runtime.scheduler.adaptivebatch.AdaptiveBatchSchedulerFactory)3