Examples with ResourceCounter org.apache.flink.runtime.util.ResourceCounter used on opensource projects

Example 31 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testDecreasingResourceRequirementsWillSendResourceRequirementNotification.

@Test
public void testDecreasingResourceRequirementsWillSendResourceRequirementNotification() throws InterruptedException {
    final NewResourceRequirementsService requirementsListener = new NewResourceRequirementsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPool(requirementsListener);
    final ResourceCounter increment = ResourceCounter.withResource(RESOURCE_PROFILE_1, 3);
    slotPool.increaseResourceRequirementsBy(increment);
    requirementsListener.takeResourceRequirements();
    final ResourceCounter decrement = ResourceCounter.withResource(RESOURCE_PROFILE_1, 2);
    slotPool.decreaseResourceRequirementsBy(decrement);
    final ResourceCounter totalResources = increment.subtract(decrement);
    assertThat(requirementsListener.takeResourceRequirements(), is(toResourceRequirements(totalResources)));
    assertThat(requirementsListener.hasNextResourceRequirements(), is(false));
}

Example 32 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testOfferingTooManySlotsWillRejectSuperfluousSlots.

@Test
public void testOfferingTooManySlotsWillRejectSuperfluousSlots() {
    final NewSlotsService notifyNewSlots = new NewSlotsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPoolWithNewSlotsListener(notifyNewSlots);
    final ResourceCounter resourceRequirements = createResourceRequirements();
    slotPool.increaseResourceRequirementsBy(resourceRequirements);
    final ResourceCounter increasedRequirements = resourceRequirements.add(RESOURCE_PROFILE_1, 2);
    final Collection<SlotOffer> slotOffers = createSlotOffersForResourceRequirements(increasedRequirements);
    final Collection<SlotOffer> acceptedSlots = SlotPoolTestUtils.offerSlots(slotPool, slotOffers);
    final Map<ResourceProfile, Long> resourceProfileCount = acceptedSlots.stream().map(SlotOffer::getResourceProfile).collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
    for (Map.Entry<ResourceProfile, Integer> resourceCount : resourceRequirements.getResourcesWithCount()) {
        assertThat(resourceProfileCount.getOrDefault(resourceCount.getKey(), 0L), is((long) resourceCount.getValue()));
    }
}

Example 33 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testFreedSlotWillBeUsedToFulfillOutstandingResourceRequirementsOfSameProfile.

@Test
public void testFreedSlotWillBeUsedToFulfillOutstandingResourceRequirementsOfSameProfile() throws InterruptedException {
    final NewSlotsService notifyNewSlots = new NewSlotsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPoolWithNewSlotsListener(notifyNewSlots);
    final ResourceCounter initialRequirements = ResourceCounter.withResource(RESOURCE_PROFILE_1, 1);
    increaseRequirementsAndOfferSlotsToSlotPool(slotPool, initialRequirements, null);
    final Collection<PhysicalSlot> newSlots = drainNewSlotService(notifyNewSlots);
    final PhysicalSlot newSlot = Iterables.getOnlyElement(newSlots);
    slotPool.reserveFreeSlot(newSlot.getAllocationId(), RESOURCE_PROFILE_1);
    slotPool.freeReservedSlot(newSlot.getAllocationId(), null, 0);
    final Collection<PhysicalSlot> recycledSlots = drainNewSlotService(notifyNewSlots);
    assertThat(Iterables.getOnlyElement(recycledSlots), sameInstance(newSlot));
    final Collection<SlotOffer> newSlotOffers = createSlotOffersForResourceRequirements(initialRequirements);
    // the pending requirement should be fulfilled by the freed slot --> rejecting new slot
    // offers
    final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(newSlotOffers, new LocalTaskManagerLocation(), SlotPoolTestUtils.createTaskManagerGateway(null), 0);
    assertThat(acceptedSlots, is(empty()));
    assertTrue(slotPool.calculateUnfulfilledResources().isEmpty());
}

Example 34 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testOnlyReturnExcessIdleSlots.

@Test
public void testOnlyReturnExcessIdleSlots() {
    final Time idleSlotTimeout = Time.seconds(10);
    final long offerTime = 0;
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().setIdleSlotTimeout(idleSlotTimeout).build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final Collection<SlotOffer> slotOffers = createSlotOffersForResourceRequirements(resourceRequirements);
    slotPool.increaseResourceRequirementsBy(resourceRequirements);
    final Collection<SlotOffer> acceptedSlots = SlotPoolTestUtils.offerSlots(slotPool, slotOffers);
    final ResourceCounter requiredResources = ResourceCounter.withResource(RESOURCE_PROFILE_1, 1);
    final ResourceCounter excessRequirements = resourceRequirements.subtract(requiredResources);
    slotPool.decreaseResourceRequirementsBy(excessRequirements);
    slotPool.releaseIdleSlots(offerTime + idleSlotTimeout.toMilliseconds());
    assertThat(acceptedSlots, is(not(empty())));
    assertThat(slotPool.getFulfilledResourceRequirements(), is(requiredResources));
}

Example 35 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotsReturnsSlot.

@Test
public void testReleaseSlotsReturnsSlot() {
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> slotOffers = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, taskManagerLocation, testingTaskExecutorGateway);
    slotPool.releaseSlots(taskManagerLocation.getResourceID(), new FlinkException("Test failure"));
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(freedSlots, containsInAnyOrder(slotOffers.stream().map(SlotOffer::getAllocationId).toArray()));
}