Examples with DefaultDeclarativeSlotPool org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool used on opensource projects

Example 6 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testStatusMetrics.

@Test
public void testStatusMetrics() throws Exception {
    final CompletableFuture<UpTimeGauge> upTimeMetricFuture = new CompletableFuture<>();
    final CompletableFuture<DownTimeGauge> downTimeMetricFuture = new CompletableFuture<>();
    // restartingTime acts as a stand-in for generic status time metrics
    final CompletableFuture<Gauge<Long>> restartTimeMetricFuture = new CompletableFuture<>();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((metric, name, group) -> {
        switch(name) {
            case UpTimeGauge.METRIC_NAME:
                upTimeMetricFuture.complete((UpTimeGauge) metric);
                break;
            case DownTimeGauge.METRIC_NAME:
                downTimeMetricFuture.complete((DownTimeGauge) metric);
                break;
            case "restartingTimeTotal":
                restartTimeMetricFuture.complete((Gauge<Long>) metric);
                break;
        }
    }).build();
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.MIN_PARALLELISM_INCREASE, 1);
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(10L));
    configuration.set(MetricOptions.JOB_STATUS_METRICS, Arrays.asList(MetricOptions.JobStatusMetrics.TOTAL_TIME));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobManagerJobMetricGroup(JobManagerMetricGroup.createJobManagerMetricGroup(metricRegistry, "localhost").addJob(new JobID(), "jobName")).setDeclarativeSlotPool(declarativeSlotPool).build();
    final UpTimeGauge upTimeGauge = upTimeMetricFuture.get();
    final DownTimeGauge downTimeGauge = downTimeMetricFuture.get();
    final Gauge<Long> restartTimeGauge = restartTimeMetricFuture.get();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + PARALLELISM);
    taskManagerGateway.setCancelConsumer(createCancelConsumer(scheduler));
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // wait for the first task submission
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    // sleep a bit to ensure uptime is > 0
    Thread.sleep(10L);
    assertThat(upTimeGauge.getValue()).isGreaterThan(0L);
    assertThat(downTimeGauge.getValue()).isEqualTo(0L);
    assertThat(restartTimeGauge.getValue()).isEqualTo(0L);
    singleThreadMainThreadExecutor.execute(() -> {
        // offer more slots, which will cause a restart in order to scale up
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // wait for the second task submissions
    taskManagerGateway.waitForSubmissions(2, Duration.ofSeconds(5));
    // sleep a bit to ensure uptime is > 0
    Thread.sleep(10L);
    assertThat(upTimeGauge.getValue()).isGreaterThan(0L);
    assertThat(downTimeGauge.getValue()).isEqualTo(0L);
    // can be zero if the restart is very quick
    assertThat(restartTimeGauge.getValue()).isGreaterThanOrEqualTo(0L);
}

Example 7 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testStartSchedulingSetsResourceRequirementsForReactiveMode.

@Test
public void testStartSchedulingSetsResourceRequirementsForReactiveMode() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.SCHEDULER_MODE, SchedulerExecutionMode.REACTIVE);
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).setJobMasterConfiguration(configuration).build();
    scheduler.startScheduling();
    // should request the max possible resources
    final int expectedParallelism = KeyGroupRangeAssignment.computeDefaultMaxParallelism(PARALLELISM);
    assertThat(declarativeSlotPool.getResourceRequirements()).contains(ResourceRequirement.create(ResourceProfile.UNKNOWN, expectedParallelism));
}

Example 8 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testStartSchedulingSetsResourceRequirementsForDefaultMode.

@Test
public void testStartSchedulingSetsResourceRequirementsForDefaultMode() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).build();
    scheduler.startScheduling();
    assertThat(declarativeSlotPool.getResourceRequirements()).contains(ResourceRequirement.create(ResourceProfile.UNKNOWN, PARALLELISM));
}

Example 9 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testNumRestartsMetric.

@Test
public void testNumRestartsMetric() throws Exception {
    final CompletableFuture<Gauge<Long>> numRestartsMetricFuture = new CompletableFuture<>();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((metric, name, group) -> {
        if (MetricNames.NUM_RESTARTS.equals(name)) {
            numRestartsMetricFuture.complete((Gauge<Long>) metric);
        }
    }).build();
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = new DefaultDeclarativeSlotPool(jobGraph.getJobID(), new DefaultAllocatedSlotPool(), ignored -> {
    }, Time.minutes(10), Time.minutes(10));
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.MIN_PARALLELISM_INCREASE, 1);
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobManagerJobMetricGroup(JobManagerMetricGroup.createJobManagerMetricGroup(metricRegistry, "localhost").addJob(new JobID(), "jobName")).setDeclarativeSlotPool(declarativeSlotPool).build();
    final Gauge<Long> numRestartsMetric = numRestartsMetricFuture.get();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + PARALLELISM);
    taskManagerGateway.setCancelConsumer(createCancelConsumer(scheduler));
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        declarativeSlotPool.offerSlots(createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), new LocalTaskManagerLocation(), taskManagerGateway, System.currentTimeMillis());
    });
    // wait for the first task submission
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    assertThat(numRestartsMetric.getValue()).isEqualTo(0L);
    singleThreadMainThreadExecutor.execute(() -> {
        // offer more slots, which will cause a restart in order to scale up
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, PARALLELISM)), taskManagerGateway);
    });
    // wait for the second task submissions
    taskManagerGateway.waitForSubmissions(PARALLELISM, Duration.ofSeconds(5));
    assertThat(numRestartsMetric.getValue()).isEqualTo(1L);
}

Example 10 with DefaultDeclarativeSlotPool

use of org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool in project flink by apache.

the class AdaptiveSchedulerTest method testHasEnoughResourcesReturnsTrueIfSatisfied.

@Test
public void testHasEnoughResourcesReturnsTrueIfSatisfied() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).setDeclarativeSlotPool(declarativeSlotPool).build();
    scheduler.startScheduling();
    final ResourceCounter resourceRequirement = ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1);
    offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(resourceRequirement));
    assertThat(scheduler.hasDesiredResources(resourceRequirement)).isTrue();
}