Examples with DefaultDeclarativeSlotPool org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool used on opensource projects

Example 1 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testConsistentMaxParallelism.

@Test
public void testConsistentMaxParallelism() throws Exception {
    final int parallelism = 240;
    final int expectedMaxParallelism = KeyGroupRangeAssignment.computeDefaultMaxParallelism(parallelism);
    final JobVertex vertex = createNoOpVertex(parallelism);
    final JobGraph jobGraph = streamingJobGraph(vertex);
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).setJobMasterConfiguration(configuration).build();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + parallelism);
    taskManagerGateway.setCancelConsumer(createCancelConsumer(scheduler));
    // offer just enough resources to run at the lowest possible parallelism
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // Wait for task to be submitted
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    ArchivedExecutionGraph executionGraph = getArchivedExecutionGraphForRunningJob(scheduler).get();
    ArchivedExecutionJobVertex archivedVertex = executionGraph.getJobVertex(vertex.getID());
    // ensure that the parallelism was submitted based on what is available
    assertThat(archivedVertex.getParallelism()).isEqualTo(1);
    // and that the max parallelism was submitted based on what was configured
    assertThat(archivedVertex.getMaxParallelism()).isEqualTo(expectedMaxParallelism);
    // offer the resources to run at full parallelism
    singleThreadMainThreadExecutor.execute(() -> {
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, parallelism)), taskManagerGateway);
    });
    // wait for the job to be re-submitted
    taskManagerGateway.waitForSubmissions(parallelism, Duration.ofSeconds(5));
    ArchivedExecutionGraph resubmittedExecutionGraph = getArchivedExecutionGraphForRunningJob(scheduler).get();
    ArchivedExecutionJobVertex resubmittedArchivedVertex = resubmittedExecutionGraph.getJobVertex(vertex.getID());
    // ensure that the parallelism was submitted based on what is available
    assertThat(resubmittedArchivedVertex.getParallelism()).isEqualTo(parallelism);
    // and that the max parallelism was submitted based on what was configured
    assertThat(resubmittedArchivedVertex.getMaxParallelism()).isEqualTo(expectedMaxParallelism);
}

Example 2 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testJobStatusListenerNotifiedOfJobStatusChanges.

@Test
public void testJobStatusListenerNotifiedOfJobStatusChanges() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final CompletableFuture<Void> jobCreatedNotification = new CompletableFuture<>();
    final CompletableFuture<Void> jobRunningNotification = new CompletableFuture<>();
    final CompletableFuture<Void> jobFinishedNotification = new CompletableFuture<>();
    final CompletableFuture<JobStatus> unexpectedJobStatusNotification = new CompletableFuture<>();
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobStatusListener((jobId, newJobStatus, timestamp) -> {
        switch(newJobStatus) {
            case CREATED:
                jobCreatedNotification.complete(null);
                break;
            case RUNNING:
                jobRunningNotification.complete(null);
                break;
            case FINISHED:
                jobFinishedNotification.complete(null);
                break;
            default:
                unexpectedJobStatusNotification.complete(newJobStatus);
        }
    }).setDeclarativeSlotPool(declarativeSlotPool).build();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + PARALLELISM);
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // wait for the task submission
    final TaskDeploymentDescriptor submittedTask = taskManagerGateway.submittedTasks.take();
    // let the job finish
    singleThreadMainThreadExecutor.execute(() -> scheduler.updateTaskExecutionState(new TaskExecutionState(submittedTask.getExecutionAttemptId(), ExecutionState.FINISHED)));
    jobCreatedNotification.get();
    jobRunningNotification.get();
    jobFinishedNotification.get();
    assertThat(unexpectedJobStatusNotification.isDone()).isFalse();
}

Example 3 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testHasEnoughResourcesUsesUnmatchedSlotsAsUnknown.

@Test
public void testHasEnoughResourcesUsesUnmatchedSlotsAsUnknown() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).build();
    scheduler.startScheduling();
    final int numRequiredSlots = 1;
    final ResourceCounter requiredResources = ResourceCounter.withResource(ResourceProfile.UNKNOWN, numRequiredSlots);
    final ResourceCounter providedResources = ResourceCounter.withResource(ResourceProfile.newBuilder().setCpuCores(1).build(), numRequiredSlots);
    offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(providedResources));
    assertThat(scheduler.hasDesiredResources(requiredResources)).isTrue();
}

Example 4 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testResourceAcquisitionTriggersJobExecution.

/**
 * Tests that the listener for new slots is properly set up.
 */
@Test
public void testResourceAcquisitionTriggersJobExecution() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).setJobMasterConfiguration(configuration).build();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(PARALLELISM);
    CompletableFuture<State> startingStateFuture = new CompletableFuture<>();
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        startingStateFuture.complete(scheduler.getState());
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, PARALLELISM)), taskManagerGateway);
    });
    assertThat(startingStateFuture.get()).isInstanceOf(WaitingForResources.class);
    // Wait for all tasks to be submitted
    taskManagerGateway.waitForSubmissions(PARALLELISM, Duration.ofSeconds(5));
    final ArchivedExecutionGraph executionGraph = CompletableFuture.supplyAsync(() -> scheduler.requestJob().getArchivedExecutionGraph(), singleThreadMainThreadExecutor).get();
    assertThat(executionGraph.getJobVertex(JOB_VERTEX.getID()).getParallelism()).isEqualTo(PARALLELISM);
}

Example 5 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testExecutionGraphGenerationSetsInitializationTimestamp.

@Test
public void testExecutionGraphGenerationSetsInitializationTimestamp() throws Exception {
    final long initializationTimestamp = 42L;
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final AdaptiveScheduler adaptiveScheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setInitializationTimestamp(initializationTimestamp).setDeclarativeSlotPool(declarativeSlotPool).setJobMasterConfiguration(configuration).build();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(PARALLELISM);
    singleThreadMainThreadExecutor.execute(() -> {
        adaptiveScheduler.startScheduling();
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, PARALLELISM)), taskManagerGateway);
    });
    // Wait for just the first submission to indicate the execution graph is ready
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    final ArchivedExecutionGraph executionGraph = CompletableFuture.supplyAsync(() -> adaptiveScheduler.requestJob().getArchivedExecutionGraph(), singleThreadMainThreadExecutor).join();
    assertThat(executionGraph.getStatusTimestamp(JobStatus.INITIALIZING)).isEqualTo(initializationTimestamp);
}