Examples with ResourceCounter org.apache.flink.runtime.util.ResourceCounter used on opensource projects

Example 11 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotReturnsSlot.

@Test
public void testReleaseSlotReturnsSlot() throws InterruptedException {
    final NewSlotsService notifyNewSlots = new NewSlotsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPoolWithNewSlotsListener(notifyNewSlots);
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, new LocalTaskManagerLocation(), testingTaskExecutorGateway);
    final Collection<? extends PhysicalSlot> physicalSlots = notifyNewSlots.takeNewSlots();
    final PhysicalSlot physicalSlot = physicalSlots.iterator().next();
    slotPool.releaseSlot(physicalSlot.getAllocationId(), new FlinkException("Test failure"));
    final AllocationID freedSlot = Iterables.getOnlyElement(freeSlotConsumer.drainFreedSlots());
    assertThat(freedSlot, is(physicalSlot.getAllocationId()));
}

Example 12 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReserveFreeSlotForResourceUpdatesAvailableResourcesAndRequirements.

@Test
public void testReserveFreeSlotForResourceUpdatesAvailableResourcesAndRequirements() {
    final DefaultDeclarativeSlotPool slotPool = new DefaultDeclarativeSlotPoolBuilder().build();
    final ResourceProfile largeResourceProfile = ResourceProfile.newBuilder().setManagedMemoryMB(1024).build();
    final ResourceProfile smallResourceProfile = ResourceProfile.UNKNOWN;
    slotPool.increaseResourceRequirementsBy(ResourceCounter.withResource(largeResourceProfile, 1));
    SlotPoolTestUtils.offerSlots(slotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(largeResourceProfile, 1)));
    slotPool.increaseResourceRequirementsBy(ResourceCounter.withResource(smallResourceProfile, 1));
    final SlotInfoWithUtilization largeSlot = slotPool.getFreeSlotsInformation().stream().filter(slot -> slot.getResourceProfile().equals(largeResourceProfile)).findFirst().get();
    slotPool.reserveFreeSlot(largeSlot.getAllocationId(), smallResourceProfile);
    ResourceCounter availableResources = slotPool.getFulfilledResourceRequirements();
    assertThat(availableResources.getResourceCount(smallResourceProfile), is(1));
    assertThat(availableResources.getResourceCount(largeResourceProfile), is(0));
    Collection<ResourceRequirement> currentResourceRequirements = slotPool.getResourceRequirements();
    // since we used one of the large slots for fulfilling another profile, we now need another
    // large slot for fulfill the original requirement
    // conversely we no longer need the small slot, because we are now using another slot for it
    assertThat(currentResourceRequirements, hasItems(ResourceRequirement.create(largeResourceProfile, 2)));
}

Example 13 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testSetResourceRequirementsOverwritesPreviousValue.

@Test
public void testSetResourceRequirementsOverwritesPreviousValue() {
    final DefaultDeclarativeSlotPool slotPool = new DefaultDeclarativeSlotPoolBuilder().build();
    slotPool.setResourceRequirements(ResourceCounter.withResource(RESOURCE_PROFILE_1, 1));
    final ResourceCounter resourceRequirements = ResourceCounter.withResource(RESOURCE_PROFILE_2, 1);
    slotPool.setResourceRequirements(resourceRequirements);
    assertThat(slotPool.getResourceRequirements(), is(toResourceRequirements(resourceRequirements)));
}

Example 14 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class FineGrainedTaskManagerTrackerTest method testRecordPendingAllocations.

@Test
public void testRecordPendingAllocations() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final PendingTaskManager pendingTaskManager1 = new PendingTaskManager(ResourceProfile.ANY, 1);
    final PendingTaskManager pendingTaskManager2 = new PendingTaskManager(ResourceProfile.ANY, 1);
    final JobID jobId = new JobID();
    final ResourceCounter resourceCounter = ResourceCounter.withResource(ResourceProfile.ANY, 1);
    taskManagerTracker.addPendingTaskManager(pendingTaskManager1);
    taskManagerTracker.addPendingTaskManager(pendingTaskManager2);
    taskManagerTracker.replaceAllPendingAllocations(Collections.singletonMap(pendingTaskManager1.getPendingTaskManagerId(), Collections.singletonMap(jobId, resourceCounter)));
    // Only the last time is recorded
    taskManagerTracker.replaceAllPendingAllocations(Collections.singletonMap(pendingTaskManager2.getPendingTaskManagerId(), Collections.singletonMap(jobId, resourceCounter)));
    assertThat(taskManagerTracker.getPendingAllocationsOfPendingTaskManager(pendingTaskManager1.getPendingTaskManagerId()).size(), is(0));
    assertTrue(taskManagerTracker.getPendingAllocationsOfPendingTaskManager(pendingTaskManager2.getPendingTaskManagerId()).containsKey(jobId));
    assertThat(taskManagerTracker.getPendingAllocationsOfPendingTaskManager(pendingTaskManager2.getPendingTaskManagerId()).get(jobId).getResourceCount(ResourceProfile.ANY), is(1));
}

Example 15 with ResourceCounter

use of org.apache.flink.runtime.util.ResourceCounter in project flink by apache.

the class DefaultResourceAllocationStrategyTest method testFulfillRequirementWithPendingResources.

@Test
public void testFulfillRequirementWithPendingResources() {
    final JobID jobId = new JobID();
    final List<ResourceRequirement> requirements = new ArrayList<>();
    final ResourceProfile largeResource = DEFAULT_SLOT_RESOURCE.multiply(3);
    final PendingTaskManager pendingTaskManager = new PendingTaskManager(DEFAULT_SLOT_RESOURCE.multiply(NUM_OF_SLOTS), NUM_OF_SLOTS);
    final TaskManagerResourceInfoProvider taskManagerResourceInfoProvider = TestingTaskManagerResourceInfoProvider.newBuilder().setPendingTaskManagersSupplier(() -> Collections.singleton(pendingTaskManager)).build();
    requirements.add(ResourceRequirement.create(largeResource, 2));
    requirements.add(ResourceRequirement.create(ResourceProfile.UNKNOWN, 4));
    final ResourceAllocationResult result = STRATEGY.tryFulfillRequirements(Collections.singletonMap(jobId, requirements), taskManagerResourceInfoProvider);
    assertThat(result.getUnfulfillableJobs(), is(empty()));
    assertThat(result.getAllocationsOnRegisteredResources().keySet(), is(empty()));
    assertThat(result.getPendingTaskManagersToAllocate().size(), is(1));
    final PendingTaskManagerId newAllocated = result.getPendingTaskManagersToAllocate().get(0).getPendingTaskManagerId();
    ResourceCounter allFulfilledRequirements = ResourceCounter.empty();
    for (Map.Entry<ResourceProfile, Integer> resourceWithCount : result.getAllocationsOnPendingResources().get(pendingTaskManager.getPendingTaskManagerId()).get(jobId).getResourcesWithCount()) {
        allFulfilledRequirements = allFulfilledRequirements.add(resourceWithCount.getKey(), resourceWithCount.getValue());
    }
    for (Map.Entry<ResourceProfile, Integer> resourceWithCount : result.getAllocationsOnPendingResources().get(newAllocated).get(jobId).getResourcesWithCount()) {
        allFulfilledRequirements = allFulfilledRequirements.add(resourceWithCount.getKey(), resourceWithCount.getValue());
    }
    assertThat(allFulfilledRequirements.getResourceCount(DEFAULT_SLOT_RESOURCE), is(4));
    assertThat(allFulfilledRequirements.getResourceCount(largeResource), is(2));
}