Examples with DefaultSchedulerFactory org.apache.flink.runtime.scheduler.DefaultSchedulerFactory used on opensource projects

Example 1 with DefaultSchedulerFactory

use of org.apache.flink.runtime.scheduler.DefaultSchedulerFactory in project flink by apache.

the class DefaultSlotPoolServiceSchedulerFactory method fromConfiguration.

public static DefaultSlotPoolServiceSchedulerFactory fromConfiguration(Configuration configuration, JobType jobType) {
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT));
    final Time batchSlotTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
    final SlotPoolServiceFactory slotPoolServiceFactory;
    final SchedulerNGFactory schedulerNGFactory;
    JobManagerOptions.SchedulerType schedulerType = ClusterOptions.getSchedulerType(configuration);
    if (schedulerType == JobManagerOptions.SchedulerType.Adaptive && jobType == JobType.BATCH) {
        LOG.info("Adaptive Scheduler configured, but Batch job detected. Changing scheduler type to NG / DefaultScheduler.");
        // overwrite
        schedulerType = JobManagerOptions.SchedulerType.Ng;
    }
    switch(schedulerType) {
        case Ng:
            schedulerNGFactory = new DefaultSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        case Adaptive:
            schedulerNGFactory = getAdaptiveSchedulerFactoryFromConfiguration(configuration);
            slotPoolServiceFactory = new DeclarativeSlotPoolServiceFactory(SystemClock.getInstance(), slotIdleTimeout, rpcTimeout);
            break;
        case AdaptiveBatch:
            schedulerNGFactory = new AdaptiveBatchSchedulerFactory();
            slotPoolServiceFactory = new DeclarativeSlotPoolBridgeServiceFactory(SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, batchSlotTimeout, getRequestSlotMatchingStrategy(configuration, jobType));
            break;
        default:
            throw new IllegalArgumentException(String.format("Illegal value [%s] for config option [%s]", schedulerType, JobManagerOptions.SCHEDULER.key()));
    }
    return new DefaultSlotPoolServiceSchedulerFactory(slotPoolServiceFactory, schedulerNGFactory);
}

Example 2 with DefaultSchedulerFactory

use of org.apache.flink.runtime.scheduler.DefaultSchedulerFactory in project flink by apache.

the class JobMasterTest method testAllocatedSlotReportDoesNotContainStaleInformation.

/**
 * Tests that the {@link AllocatedSlotReport} contains up to date information and not stale
 * information about the allocated slots on the {@link JobMaster}.
 *
 * <p>This is a probabilistic test case which only fails if executed repeatedly without the fix
 * for FLINK-12863.
 */
@Test
public void testAllocatedSlotReportDoesNotContainStaleInformation() throws Exception {
    final CompletableFuture<Void> assertionFuture = new CompletableFuture<>();
    final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    final AtomicBoolean terminateHeartbeatVerification = new AtomicBoolean(false);
    final OneShotLatch hasReceivedSlotOffers = new OneShotLatch();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setHeartbeatJobManagerFunction((taskManagerId, allocatedSlotReport) -> {
        try {
            if (hasReceivedSlotOffers.isTriggered()) {
                assertThat(allocatedSlotReport.getAllocatedSlotInfos(), hasSize(1));
            } else {
                assertThat(allocatedSlotReport.getAllocatedSlotInfos(), empty());
            }
        } catch (AssertionError e) {
            assertionFuture.completeExceptionally(e);
        }
        if (terminateHeartbeatVerification.get()) {
            assertionFuture.complete(null);
        }
        return FutureUtils.completedVoidFuture();
    }).createTestingTaskExecutorGateway();
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    final JobManagerSharedServices jobManagerSharedServices = new TestingJobManagerSharedServicesBuilder().build();
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withHeartbeatServices(new HeartbeatServices(5L, 1000L)).withSlotPoolServiceSchedulerFactory(DefaultSlotPoolServiceSchedulerFactory.create(new TestingSlotPoolFactory(hasReceivedSlotOffers), new DefaultSchedulerFactory())).createJobMaster();
    jobMaster.start();
    try {
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        // register task manager will trigger monitor heartbeat target, schedule heartbeat
        // request at interval time
        CompletableFuture<RegistrationResponse> registrationResponse = jobMasterGateway.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(taskExecutorGateway.getAddress(), unresolvedTaskManagerLocation, TestingUtils.zeroUUID()), testingTimeout);
        // wait for the completion of the registration
        registrationResponse.get();
        final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
        final CompletableFuture<Collection<SlotOffer>> slotOfferFuture = jobMasterGateway.offerSlots(unresolvedTaskManagerLocation.getResourceID(), Collections.singleton(slotOffer), testingTimeout);
        assertThat(slotOfferFuture.get(), containsInAnyOrder(slotOffer));
        terminateHeartbeatVerification.set(true);
        // make sure that no assertion has been violated
        assertionFuture.get();
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
        jobManagerSharedServices.shutdown();
    }
}