Examples with LocalTaskManagerLocation org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation used on opensource projects

Example 26 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DeclarativeSlotPoolBridgePreferredAllocationsTest method testDeclarativeSlotPoolTakesPreferredAllocationsIntoAccount.

@Test
public void testDeclarativeSlotPoolTakesPreferredAllocationsIntoAccount() throws Exception {
    final DeclarativeSlotPoolBridge declarativeSlotPoolBridge = new DeclarativeSlotPoolBridge(new JobID(), new DefaultDeclarativeSlotPoolFactory(), SystemClock.getInstance(), TestingUtils.infiniteTime(), TestingUtils.infiniteTime(), TestingUtils.infiniteTime(), PreferredAllocationRequestSlotMatchingStrategy.INSTANCE);
    declarativeSlotPoolBridge.start(JobMasterId.generate(), "localhost", ComponentMainThreadExecutorServiceAdapter.forMainThread());
    final LocalTaskManagerLocation localTaskManagerLocation = new LocalTaskManagerLocation();
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final CompletableFuture<PhysicalSlot> slotRequestWithPreferredAllocation1 = requestSlot(declarativeSlotPoolBridge, Collections.singleton(allocationId1));
    final CompletableFuture<PhysicalSlot> slotRequestWithEmptyPreferredAllocations = requestSlot(declarativeSlotPoolBridge, Collections.emptySet());
    final CompletableFuture<PhysicalSlot> slotRequestWithPreferredAllocation2 = requestSlot(declarativeSlotPoolBridge, Collections.singleton(allocationId2));
    final Collection<SlotOffer> slotOffers = new ArrayList<>();
    slotOffers.add(new SlotOffer(allocationId2, 0, ResourceProfile.ANY));
    final AllocationID otherAllocationId = new AllocationID();
    slotOffers.add(new SlotOffer(otherAllocationId, 1, ResourceProfile.ANY));
    slotOffers.add(new SlotOffer(allocationId1, 2, ResourceProfile.ANY));
    declarativeSlotPoolBridge.registerTaskManager(localTaskManagerLocation.getResourceID());
    declarativeSlotPoolBridge.offerSlots(localTaskManagerLocation, new SimpleAckingTaskManagerGateway(), slotOffers);
    assertThat(slotRequestWithPreferredAllocation1.join().getAllocationId()).isEqualTo(allocationId1);
    assertThat(slotRequestWithPreferredAllocation2.join().getAllocationId()).isEqualTo(allocationId2);
    assertThat(slotRequestWithEmptyPreferredAllocations.join().getAllocationId()).isEqualTo(otherAllocationId);
}

Example 27 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReturnIdleSlotsAfterTimeout.

@Test
public void testReturnIdleSlotsAfterTimeout() {
    final Time idleSlotTimeout = Time.seconds(10);
    final long offerTime = 0;
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().setIdleSlotTimeout(idleSlotTimeout).build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> acceptedSlots = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, new LocalTaskManagerLocation(), testingTaskExecutorGateway);
    // decrease the resource requirements so that slots are no longer needed
    slotPool.decreaseResourceRequirementsBy(resourceRequirements);
    slotPool.releaseIdleSlots(offerTime + idleSlotTimeout.toMilliseconds());
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(acceptedSlots, is(not(empty())));
    assertThat(freedSlots, containsInAnyOrder(acceptedSlots.stream().map(SlotOffer::getAllocationId).toArray()));
    assertNoAvailableAndRequiredResources(slotPool);
}

Example 28 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testFreedSlotWillBeUsedToFulfillOutstandingResourceRequirementsOfSameProfile.

@Test
public void testFreedSlotWillBeUsedToFulfillOutstandingResourceRequirementsOfSameProfile() throws InterruptedException {
    final NewSlotsService notifyNewSlots = new NewSlotsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPoolWithNewSlotsListener(notifyNewSlots);
    final ResourceCounter initialRequirements = ResourceCounter.withResource(RESOURCE_PROFILE_1, 1);
    increaseRequirementsAndOfferSlotsToSlotPool(slotPool, initialRequirements, null);
    final Collection<PhysicalSlot> newSlots = drainNewSlotService(notifyNewSlots);
    final PhysicalSlot newSlot = Iterables.getOnlyElement(newSlots);
    slotPool.reserveFreeSlot(newSlot.getAllocationId(), RESOURCE_PROFILE_1);
    slotPool.freeReservedSlot(newSlot.getAllocationId(), null, 0);
    final Collection<PhysicalSlot> recycledSlots = drainNewSlotService(notifyNewSlots);
    assertThat(Iterables.getOnlyElement(recycledSlots), sameInstance(newSlot));
    final Collection<SlotOffer> newSlotOffers = createSlotOffersForResourceRequirements(initialRequirements);
    // the pending requirement should be fulfilled by the freed slot --> rejecting new slot
    // offers
    final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(newSlotOffers, new LocalTaskManagerLocation(), SlotPoolTestUtils.createTaskManagerGateway(null), 0);
    assertThat(acceptedSlots, is(empty()));
    assertTrue(slotPool.calculateUnfulfilledResources().isEmpty());
}

Example 29 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotsReturnsSlot.

@Test
public void testReleaseSlotsReturnsSlot() {
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> slotOffers = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, taskManagerLocation, testingTaskExecutorGateway);
    slotPool.releaseSlots(taskManagerLocation.getResourceID(), new FlinkException("Test failure"));
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(freedSlots, containsInAnyOrder(slotOffers.stream().map(SlotOffer::getAllocationId).toArray()));
}

Example 30 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class AdaptiveSchedulerTest method testNumRestartsMetric.

@Test
public void testNumRestartsMetric() throws Exception {
    final CompletableFuture<Gauge<Long>> numRestartsMetricFuture = new CompletableFuture<>();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((metric, name, group) -> {
        if (MetricNames.NUM_RESTARTS.equals(name)) {
            numRestartsMetricFuture.complete((Gauge<Long>) metric);
        }
    }).build();
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = new DefaultDeclarativeSlotPool(jobGraph.getJobID(), new DefaultAllocatedSlotPool(), ignored -> {
    }, Time.minutes(10), Time.minutes(10));
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.MIN_PARALLELISM_INCREASE, 1);
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobManagerJobMetricGroup(JobManagerMetricGroup.createJobManagerMetricGroup(metricRegistry, "localhost").addJob(new JobID(), "jobName")).setDeclarativeSlotPool(declarativeSlotPool).build();
    final Gauge<Long> numRestartsMetric = numRestartsMetricFuture.get();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + PARALLELISM);
    taskManagerGateway.setCancelConsumer(createCancelConsumer(scheduler));
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        declarativeSlotPool.offerSlots(createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), new LocalTaskManagerLocation(), taskManagerGateway, System.currentTimeMillis());
    });
    // wait for the first task submission
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    assertThat(numRestartsMetric.getValue()).isEqualTo(0L);
    singleThreadMainThreadExecutor.execute(() -> {
        // offer more slots, which will cause a restart in order to scale up
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, PARALLELISM)), taskManagerGateway);
    });
    // wait for the second task submissions
    taskManagerGateway.waitForSubmissions(PARALLELISM, Duration.ofSeconds(5));
    assertThat(numRestartsMetric.getValue()).isEqualTo(1L);
}