Examples with ResourceRequirement org.apache.flink.runtime.slots.ResourceRequirement used on opensource projects

Example 11 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceAllocationStrategy method tryFulfillRequirementsForJobWithPendingResources.

private void tryFulfillRequirementsForJobWithPendingResources(JobID jobId, Collection<ResourceRequirement> unfulfilledRequirements, List<InternalResourceInfo> availableResources, ResourceAllocationResult.Builder resultBuilder) {
    for (ResourceRequirement missingResource : unfulfilledRequirements) {
        // for this strategy, all pending resources should have the same default slot resource
        final ResourceProfile effectiveProfile = getEffectiveResourceProfile(missingResource.getResourceProfile(), defaultSlotResourceProfile);
        int numUnfulfilled = tryFulfilledRequirementWithResource(availableResources, missingResource.getNumberOfRequiredSlots(), missingResource.getResourceProfile(), jobId);
        if (!totalResourceProfile.allFieldsNoLessThan(effectiveProfile)) {
            // Can not fulfill this resource type will the default worker.
            resultBuilder.addUnfulfillableJob(jobId);
            continue;
        }
        while (numUnfulfilled > 0) {
            // Circularly add new pending task manager
            final PendingTaskManager newPendingTaskManager = new PendingTaskManager(totalResourceProfile, numSlotsPerWorker);
            resultBuilder.addPendingTaskManagerAllocate(newPendingTaskManager);
            ResourceProfile remainResource = totalResourceProfile;
            while (numUnfulfilled > 0 && canFulfillRequirement(effectiveProfile, remainResource)) {
                numUnfulfilled--;
                resultBuilder.addAllocationOnPendingResource(jobId, newPendingTaskManager.getPendingTaskManagerId(), effectiveProfile);
                remainResource = remainResource.subtract(effectiveProfile);
            }
            if (!remainResource.equals(ResourceProfile.ZERO)) {
                availableResources.add(new InternalResourceInfo(defaultSlotResourceProfile, remainResource, (jobID, slotProfile) -> resultBuilder.addAllocationOnPendingResource(jobID, newPendingTaskManager.getPendingTaskManagerId(), slotProfile)));
            }
        }
    }
}

Example 12 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class JobScopedResourceTracker method getMissingResources.

public Collection<ResourceRequirement> getMissingResources() {
    final Collection<ResourceRequirement> missingResources = new ArrayList<>();
    for (Map.Entry<ResourceProfile, Integer> requirement : resourceRequirements.getResourcesWithCount()) {
        ResourceProfile requirementProfile = requirement.getKey();
        int numRequiredResources = requirement.getValue();
        int numAcquiredResources = resourceToRequirementMapping.getNumFulfillingResources(requirementProfile);
        if (numAcquiredResources < numRequiredResources) {
            missingResources.add(ResourceRequirement.create(requirementProfile, numRequiredResources - numAcquiredResources));
        }
    }
    return missingResources;
}

Example 13 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceTrackerTest method testGetRequiredResources.

@Test
public void testGetRequiredResources() {
    DefaultResourceTracker tracker = new DefaultResourceTracker();
    ResourceRequirement requirement1 = ResourceRequirement.create(ResourceProfile.ANY, 1);
    ResourceRequirement requirement2 = ResourceRequirement.create(ResourceProfile.ANY, 2);
    tracker.notifyResourceRequirements(JOB_ID_1, Collections.singletonList(requirement1));
    tracker.notifyResourceRequirements(JOB_ID_2, Collections.singletonList(requirement2));
    Map<JobID, Collection<ResourceRequirement>> requiredResources = tracker.getMissingResources();
    assertThat(requiredResources, IsMapContaining.hasEntry(is(JOB_ID_1), contains(requirement1)));
    assertThat(requiredResources, IsMapContaining.hasEntry(is(JOB_ID_2), contains(requirement2)));
}

Example 14 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceAllocationStrategyTest method testFulfillRequirementWithRegisteredResources.

@Test
public void testFulfillRequirementWithRegisteredResources() {
    final TaskManagerInfo taskManager = new TestingTaskManagerInfo(DEFAULT_SLOT_RESOURCE.multiply(10), DEFAULT_SLOT_RESOURCE.multiply(10), DEFAULT_SLOT_RESOURCE);
    final JobID jobId = new JobID();
    final List<ResourceRequirement> requirements = new ArrayList<>();
    final ResourceProfile largeResource = DEFAULT_SLOT_RESOURCE.multiply(8);
    final TaskManagerResourceInfoProvider taskManagerResourceInfoProvider = TestingTaskManagerResourceInfoProvider.newBuilder().setRegisteredTaskManagersSupplier(() -> Collections.singleton(taskManager)).build();
    requirements.add(ResourceRequirement.create(largeResource, 1));
    requirements.add(ResourceRequirement.create(ResourceProfile.UNKNOWN, 2));
    final ResourceAllocationResult result = STRATEGY.tryFulfillRequirements(Collections.singletonMap(jobId, requirements), taskManagerResourceInfoProvider);
    assertThat(result.getUnfulfillableJobs(), is(empty()));
    assertThat(result.getAllocationsOnPendingResources().keySet(), is(empty()));
    assertThat(result.getPendingTaskManagersToAllocate(), is(empty()));
    assertThat(result.getAllocationsOnRegisteredResources().get(jobId).get(taskManager.getInstanceId()).getResourceCount(DEFAULT_SLOT_RESOURCE), is(2));
    assertThat(result.getAllocationsOnRegisteredResources().get(jobId).get(taskManager.getInstanceId()).getResourceCount(largeResource), is(1));
}

Example 15 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceAllocationStrategyTest method testUnfulfillableRequirement.

@Test
public void testUnfulfillableRequirement() {
    final TaskManagerInfo taskManager = new TestingTaskManagerInfo(DEFAULT_SLOT_RESOURCE.multiply(NUM_OF_SLOTS), DEFAULT_SLOT_RESOURCE.multiply(NUM_OF_SLOTS), DEFAULT_SLOT_RESOURCE);
    final JobID jobId = new JobID();
    final List<ResourceRequirement> requirements = new ArrayList<>();
    final ResourceProfile unfulfillableResource = DEFAULT_SLOT_RESOURCE.multiply(8);
    final TaskManagerResourceInfoProvider taskManagerResourceInfoProvider = TestingTaskManagerResourceInfoProvider.newBuilder().setRegisteredTaskManagersSupplier(() -> Collections.singleton(taskManager)).build();
    requirements.add(ResourceRequirement.create(unfulfillableResource, 1));
    final ResourceAllocationResult result = STRATEGY.tryFulfillRequirements(Collections.singletonMap(jobId, requirements), taskManagerResourceInfoProvider);
    assertThat(result.getUnfulfillableJobs(), contains(jobId));
    assertThat(result.getPendingTaskManagersToAllocate(), is(empty()));
}