Examples with ResourceRequirement org.apache.flink.runtime.slots.ResourceRequirement used on opensource projects

Example 1 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceAllocationStrategy method tryFulfillRequirements.

@Override
public ResourceAllocationResult tryFulfillRequirements(Map<JobID, Collection<ResourceRequirement>> missingResources, TaskManagerResourceInfoProvider taskManagerResourceInfoProvider) {
    final ResourceAllocationResult.Builder resultBuilder = ResourceAllocationResult.builder();
    final List<InternalResourceInfo> registeredResources = getRegisteredResources(taskManagerResourceInfoProvider, resultBuilder);
    final List<InternalResourceInfo> pendingResources = getPendingResources(taskManagerResourceInfoProvider, resultBuilder);
    for (Map.Entry<JobID, Collection<ResourceRequirement>> resourceRequirements : missingResources.entrySet()) {
        final JobID jobId = resourceRequirements.getKey();
        final Collection<ResourceRequirement> unfulfilledJobRequirements = tryFulfillRequirementsForJobWithResources(jobId, resourceRequirements.getValue(), registeredResources);
        if (!unfulfilledJobRequirements.isEmpty()) {
            tryFulfillRequirementsForJobWithPendingResources(jobId, unfulfilledJobRequirements, pendingResources, resultBuilder);
        }
    }
    return resultBuilder.build();
}

Example 2 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class JobScopedResourceTracker method getAcquiredResources.

public Collection<ResourceRequirement> getAcquiredResources() {
    final Set<ResourceProfile> knownResourceProfiles = new HashSet<>();
    knownResourceProfiles.addAll(resourceToRequirementMapping.getAllResourceProfiles());
    knownResourceProfiles.addAll(excessResources.getResources());
    final List<ResourceRequirement> acquiredResources = new ArrayList<>();
    for (ResourceProfile knownResourceProfile : knownResourceProfiles) {
        int numTotalAcquiredResources = resourceToRequirementMapping.getNumFulfilledRequirements(knownResourceProfile) + excessResources.getResourceCount(knownResourceProfile);
        ResourceRequirement resourceRequirement = ResourceRequirement.create(knownResourceProfile, numTotalAcquiredResources);
        acquiredResources.add(resourceRequirement);
    }
    return acquiredResources;
}

Example 3 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class JobScopedResourceTracker method notifyResourceRequirements.

public void notifyResourceRequirements(Collection<ResourceRequirement> newResourceRequirements) {
    Preconditions.checkNotNull(newResourceRequirements);
    resourceRequirements = ResourceCounter.empty();
    for (ResourceRequirement newResourceRequirement : newResourceRequirements) {
        resourceRequirements = resourceRequirements.add(newResourceRequirement.getResourceProfile(), newResourceRequirement.getNumberOfRequiredSlots());
    }
    findExcessSlots();
    tryAssigningExcessSlots();
}

Example 4 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DeclarativeSlotPoolServiceTest method testConnectToResourceManagerDeclaresRequiredResources.

@Test
public void testConnectToResourceManagerDeclaresRequiredResources() throws Exception {
    final Collection<ResourceRequirement> requiredResources = Arrays.asList(ResourceRequirement.create(ResourceProfile.UNKNOWN, 2), ResourceRequirement.create(ResourceProfile.ZERO, 4));
    try (DeclarativeSlotPoolService declarativeSlotPoolService = createDeclarativeSlotPoolService(new TestingDeclarativeSlotPoolFactory(new TestingDeclarativeSlotPoolBuilder().setGetResourceRequirementsSupplier(() -> requiredResources)))) {
        final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
        final CompletableFuture<ResourceRequirements> declaredResourceRequirements = new CompletableFuture<>();
        resourceManagerGateway.setDeclareRequiredResourcesFunction((jobMasterId, resourceRequirements) -> {
            declaredResourceRequirements.complete(resourceRequirements);
            return CompletableFuture.completedFuture(Acknowledge.get());
        });
        declarativeSlotPoolService.connectToResourceManager(resourceManagerGateway);
        final ResourceRequirements resourceRequirements = declaredResourceRequirements.join();
        assertThat(resourceRequirements.getResourceRequirements(), is(requiredResources));
        assertThat(resourceRequirements.getJobId(), is(jobId));
        assertThat(resourceRequirements.getTargetAddress(), is(address));
    }
}

Example 5 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReserveFreeSlotForResourceUpdatesAvailableResourcesAndRequirements.

@Test
public void testReserveFreeSlotForResourceUpdatesAvailableResourcesAndRequirements() {
    final DefaultDeclarativeSlotPool slotPool = new DefaultDeclarativeSlotPoolBuilder().build();
    final ResourceProfile largeResourceProfile = ResourceProfile.newBuilder().setManagedMemoryMB(1024).build();
    final ResourceProfile smallResourceProfile = ResourceProfile.UNKNOWN;
    slotPool.increaseResourceRequirementsBy(ResourceCounter.withResource(largeResourceProfile, 1));
    SlotPoolTestUtils.offerSlots(slotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(largeResourceProfile, 1)));
    slotPool.increaseResourceRequirementsBy(ResourceCounter.withResource(smallResourceProfile, 1));
    final SlotInfoWithUtilization largeSlot = slotPool.getFreeSlotsInformation().stream().filter(slot -> slot.getResourceProfile().equals(largeResourceProfile)).findFirst().get();
    slotPool.reserveFreeSlot(largeSlot.getAllocationId(), smallResourceProfile);
    ResourceCounter availableResources = slotPool.getFulfilledResourceRequirements();
    assertThat(availableResources.getResourceCount(smallResourceProfile), is(1));
    assertThat(availableResources.getResourceCount(largeResourceProfile), is(0));
    Collection<ResourceRequirement> currentResourceRequirements = slotPool.getResourceRequirements();
    // since we used one of the large slots for fulfilling another profile, we now need another
    // large slot for fulfill the original requirement
    // conversely we no longer need the small slot, because we are now using another slot for it
    assertThat(currentResourceRequirements, hasItems(ResourceRequirement.create(largeResourceProfile, 2)));
}