Examples with ResourceRequirement org.apache.flink.runtime.slots.ResourceRequirement used on opensource projects

Example 6 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DeclarativeSlotManagerTest method testNotificationAboutNotEnoughResources.

private static void testNotificationAboutNotEnoughResources(boolean withNotificationGracePeriod) throws Exception {
    final JobID jobId = new JobID();
    final int numRequiredSlots = 3;
    final int numExistingSlots = 1;
    List<Tuple2<JobID, Collection<ResourceRequirement>>> notEnoughResourceNotifications = new ArrayList<>();
    ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceFunction(ignored -> false).setNotEnoughResourcesConsumer((jobId1, acquiredResources) -> notEnoughResourceNotifications.add(Tuple2.of(jobId1, acquiredResources))).build();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().buildAndStart(ResourceManagerId.generate(), new ManuallyTriggeredScheduledExecutor(), resourceManagerActions)) {
        if (withNotificationGracePeriod) {
            // this should disable notifications
            slotManager.setFailUnfulfillableRequest(false);
        }
        final ResourceID taskExecutorResourceId = ResourceID.generate();
        final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(taskExecutorResourceId, new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway());
        final SlotReport slotReport = createSlotReport(taskExecutorResourceId, numExistingSlots);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        ResourceRequirements resourceRequirements = createResourceRequirements(jobId, numRequiredSlots);
        slotManager.processResourceRequirements(resourceRequirements);
        if (withNotificationGracePeriod) {
            assertThat(notEnoughResourceNotifications, empty());
            // re-enable notifications which should also trigger another resource check
            slotManager.setFailUnfulfillableRequest(true);
        }
        assertThat(notEnoughResourceNotifications, hasSize(1));
        Tuple2<JobID, Collection<ResourceRequirement>> notification = notEnoughResourceNotifications.get(0);
        assertThat(notification.f0, is(jobId));
        assertThat(notification.f1, hasItem(ResourceRequirement.create(ResourceProfile.ANY, numExistingSlots)));
        // another slot report that does not indicate any changes should not trigger another
        // notification
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), slotReport);
        assertThat(notEnoughResourceNotifications, hasSize(1));
    }
}

Example 7 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceTrackerTest method testGetAcquiredResources.

@Test
public void testGetAcquiredResources() {
    DefaultResourceTracker tracker = new DefaultResourceTracker();
    ResourceRequirement requirement1 = ResourceRequirement.create(ResourceProfile.ANY, 1);
    ResourceRequirement requirement2 = ResourceRequirement.create(ResourceProfile.ANY, 2);
    tracker.notifyAcquiredResource(JOB_ID_1, requirement1.getResourceProfile());
    for (int x = 0; x < requirement2.getNumberOfRequiredSlots(); x++) {
        tracker.notifyAcquiredResource(JOB_ID_2, requirement2.getResourceProfile());
    }
    assertThat(tracker.getAcquiredResources(JOB_ID_1), contains(requirement1));
    assertThat(tracker.getAcquiredResources(JOB_ID_2), contains(requirement2));
    tracker.notifyLostResource(JOB_ID_1, requirement1.getResourceProfile());
    assertThat(tracker.getAcquiredResources(JOB_ID_1), empty());
}

Example 8 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class FineGrainedSlotManagerTest method testNotificationAboutNotEnoughResources.

private void testNotificationAboutNotEnoughResources(boolean withNotificationGracePeriod) throws Exception {
    final JobID jobId = new JobID();
    final List<Tuple2<JobID, Collection<ResourceRequirement>>> notEnoughResourceNotifications = new ArrayList<>();
    final CompletableFuture<Void> notifyNotEnoughResourceFuture = new CompletableFuture<>();
    new Context() {

        {
            resourceActionsBuilder.setNotEnoughResourcesConsumer((jobId1, acquiredResources) -> {
                notEnoughResourceNotifications.add(Tuple2.of(jobId1, acquiredResources));
                notifyNotEnoughResourceFuture.complete(null);
            });
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction(((jobIDCollectionMap, taskManagerResourceInfoProvider) -> ResourceAllocationResult.builder().addUnfulfillableJob(jobId).build()));
            runTest(() -> {
                if (withNotificationGracePeriod) {
                    // this should disable notifications
                    runInMainThread(() -> getSlotManager().setFailUnfulfillableRequest(false));
                }
                final ResourceRequirements resourceRequirements = createResourceRequirements(jobId, 1);
                runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements));
                if (withNotificationGracePeriod) {
                    assertFutureNotComplete(notifyNotEnoughResourceFuture);
                    assertThat(notEnoughResourceNotifications, empty());
                    // re-enable notifications which should also trigger another
                    // resource check
                    runInMainThread(() -> getSlotManager().setFailUnfulfillableRequest(true));
                }
                assertFutureCompleteAndReturn(notifyNotEnoughResourceFuture);
                assertThat(notEnoughResourceNotifications, hasSize(1));
                final Tuple2<JobID, Collection<ResourceRequirement>> notification = notEnoughResourceNotifications.get(0);
                assertThat(notification.f0, is(jobId));
            });
        }
    };
}

Example 9 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DefaultResourceAllocationStrategyTest method testFulfillRequirementWithPendingResources.

@Test
public void testFulfillRequirementWithPendingResources() {
    final JobID jobId = new JobID();
    final List<ResourceRequirement> requirements = new ArrayList<>();
    final ResourceProfile largeResource = DEFAULT_SLOT_RESOURCE.multiply(3);
    final PendingTaskManager pendingTaskManager = new PendingTaskManager(DEFAULT_SLOT_RESOURCE.multiply(NUM_OF_SLOTS), NUM_OF_SLOTS);
    final TaskManagerResourceInfoProvider taskManagerResourceInfoProvider = TestingTaskManagerResourceInfoProvider.newBuilder().setPendingTaskManagersSupplier(() -> Collections.singleton(pendingTaskManager)).build();
    requirements.add(ResourceRequirement.create(largeResource, 2));
    requirements.add(ResourceRequirement.create(ResourceProfile.UNKNOWN, 4));
    final ResourceAllocationResult result = STRATEGY.tryFulfillRequirements(Collections.singletonMap(jobId, requirements), taskManagerResourceInfoProvider);
    assertThat(result.getUnfulfillableJobs(), is(empty()));
    assertThat(result.getAllocationsOnRegisteredResources().keySet(), is(empty()));
    assertThat(result.getPendingTaskManagersToAllocate().size(), is(1));
    final PendingTaskManagerId newAllocated = result.getPendingTaskManagersToAllocate().get(0).getPendingTaskManagerId();
    ResourceCounter allFulfilledRequirements = ResourceCounter.empty();
    for (Map.Entry<ResourceProfile, Integer> resourceWithCount : result.getAllocationsOnPendingResources().get(pendingTaskManager.getPendingTaskManagerId()).get(jobId).getResourcesWithCount()) {
        allFulfilledRequirements = allFulfilledRequirements.add(resourceWithCount.getKey(), resourceWithCount.getValue());
    }
    for (Map.Entry<ResourceProfile, Integer> resourceWithCount : result.getAllocationsOnPendingResources().get(newAllocated).get(jobId).getResourcesWithCount()) {
        allFulfilledRequirements = allFulfilledRequirements.add(resourceWithCount.getKey(), resourceWithCount.getValue());
    }
    assertThat(allFulfilledRequirements.getResourceCount(DEFAULT_SLOT_RESOURCE), is(4));
    assertThat(allFulfilledRequirements.getResourceCount(largeResource), is(2));
}

Example 10 with ResourceRequirement

use of org.apache.flink.runtime.slots.ResourceRequirement in project flink by apache.

the class DeclarativeSlotManager method checkResourceRequirements.

// ---------------------------------------------------------------------------------------------
// Requirement matching
// ---------------------------------------------------------------------------------------------
/**
 * Matches resource requirements against available resources. In a first round requirements are
 * matched against free slot, and any match results in a slot allocation. The remaining
 * unfulfilled requirements are matched against pending slots, allocating more workers if no
 * matching pending slot could be found. If the requirements for a job could not be fulfilled
 * then a notification is sent to the job master informing it as such.
 *
 * <p>Performance notes: At it's core this method loops, for each job, over all free/pending
 * slots for each required slot, trying to find a matching slot. One should generally go in with
 * the assumption that this runs in numberOfJobsRequiringResources * numberOfRequiredSlots *
 * numberOfFreeOrPendingSlots. This is especially important when dealing with pending slots, as
 * matches between requirements and pending slots are not persisted and recomputed on each call.
 * This may required further refinements in the future; e.g., persisting the matches between
 * requirements and pending slots, or not matching against pending slots at all.
 *
 * <p>When dealing with unspecific resource profiles (i.e., {@link ResourceProfile#ANY}/{@link
 * ResourceProfile#UNKNOWN}), then the number of free/pending slots is not relevant because we
 * only need exactly 1 comparison to determine whether a slot can be fulfilled or not, since
 * they are all the same anyway.
 *
 * <p>When dealing with specific resource profiles things can be a lot worse, with the classical
 * cases where either no matches are found, or only at the very end of the iteration. In the
 * absolute worst case, with J jobs, requiring R slots each with a unique resource profile such
 * each pair of these profiles is not matching, and S free/pending slots that don't fulfill any
 * requirement, then this method does a total of J*R*S resource profile comparisons.
 */
private void checkResourceRequirements() {
    final Map<JobID, Collection<ResourceRequirement>> missingResources = resourceTracker.getMissingResources();
    if (missingResources.isEmpty()) {
        return;
    }
    final Map<JobID, ResourceCounter> unfulfilledRequirements = new LinkedHashMap<>();
    for (Map.Entry<JobID, Collection<ResourceRequirement>> resourceRequirements : missingResources.entrySet()) {
        final JobID jobId = resourceRequirements.getKey();
        final ResourceCounter unfulfilledJobRequirements = tryAllocateSlotsForJob(jobId, resourceRequirements.getValue());
        if (!unfulfilledJobRequirements.isEmpty()) {
            unfulfilledRequirements.put(jobId, unfulfilledJobRequirements);
        }
    }
    if (unfulfilledRequirements.isEmpty()) {
        return;
    }
    ResourceCounter pendingSlots = ResourceCounter.withResources(taskExecutorManager.getPendingTaskManagerSlots().stream().collect(Collectors.groupingBy(PendingTaskManagerSlot::getResourceProfile, Collectors.summingInt(x -> 1))));
    for (Map.Entry<JobID, ResourceCounter> unfulfilledRequirement : unfulfilledRequirements.entrySet()) {
        pendingSlots = tryFulfillRequirementsWithPendingSlots(unfulfilledRequirement.getKey(), unfulfilledRequirement.getValue().getResourcesWithCount(), pendingSlots);
    }
}