Examples with ResourceRequirements org.apache.flink.runtime.slots.ResourceRequirements used on opensource projects

Example 11 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testSpreadOutSlotAllocationStrategy.

/**
 * The spread out slot allocation strategy should spread out the allocated slots across all
 * available TaskExecutors. See FLINK-12122.
 */
@Test
public void testSpreadOutSlotAllocationStrategy() throws Exception {
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotMatchingStrategy(LeastUtilizationSlotMatchingStrategy.INSTANCE).buildAndStartWithDirectExec()) {
        final List<CompletableFuture<JobID>> requestSlotFutures = new ArrayList<>();
        final int numberTaskExecutors = 5;
        // register n TaskExecutors with 2 slots each
        for (int i = 0; i < numberTaskExecutors; i++) {
            final CompletableFuture<JobID> requestSlotFuture = new CompletableFuture<>();
            requestSlotFutures.add(requestSlotFuture);
            registerTaskExecutorWithTwoSlots(slotManager, requestSlotFuture);
        }
        final JobID jobId = new JobID();
        final ResourceRequirements resourceRequirements = createResourceRequirements(jobId, numberTaskExecutors);
        slotManager.processResourceRequirements(resourceRequirements);
        // check that every TaskExecutor has received a slot request
        final Set<JobID> jobIds = new HashSet<>(FutureUtils.combineAll(requestSlotFutures).get(10L, TimeUnit.SECONDS));
        assertThat(jobIds, hasSize(1));
        assertThat(jobIds, containsInAnyOrder(jobId));
    }
}

Example 12 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class FineGrainedSlotManagerTest method testRequirementCheckOnlyTriggeredOnce.

/**
 * Test that checkResourceRequirements will only be triggered once after multiple trigger
 * function calls.
 */
@Test
public void testRequirementCheckOnlyTriggeredOnce() throws Exception {
    new Context() {

        {
            final List<CompletableFuture<Void>> checkRequirementFutures = new ArrayList<>();
            checkRequirementFutures.add(new CompletableFuture<>());
            checkRequirementFutures.add(new CompletableFuture<>());
            final long requirementCheckDelay = 50;
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction((ignored1, ignored2) -> {
                if (checkRequirementFutures.get(0).isDone()) {
                    checkRequirementFutures.get(1).complete(null);
                } else {
                    checkRequirementFutures.get(0).complete(null);
                }
                return ResourceAllocationResult.builder().build();
            });
            setRequirementCheckDelay(requirementCheckDelay);
            runTest(() -> {
                final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
                final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
                final ResourceRequirements resourceRequirements3 = createResourceRequirementsForSingleSlot();
                final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection();
                final CompletableFuture<Void> registrationFuture = new CompletableFuture<>();
                final long start = System.nanoTime();
                runInMainThread(() -> {
                    getSlotManager().processResourceRequirements(resourceRequirements1);
                    getSlotManager().processResourceRequirements(resourceRequirements2);
                    getSlotManager().registerTaskManager(taskExecutionConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                    registrationFuture.complete(null);
                });
                assertFutureCompleteAndReturn(registrationFuture);
                final long registrationTime = (System.nanoTime() - start) / 1_000_000;
                assumeTrue("The time of process requirement and register task manager must not take longer than the requirement check delay. If it does, then this indicates a very slow machine.", registrationTime < requirementCheckDelay);
                assertFutureCompleteAndReturn(checkRequirementFutures.get(0));
                assertFutureNotComplete(checkRequirementFutures.get(1));
                // checkTimes will not increase when there's no events
                Thread.sleep(requirementCheckDelay * 2);
                assertFutureNotComplete(checkRequirementFutures.get(1));
                // checkTimes will increase again if there's another
                // processResourceRequirements
                runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements3));
                assertFutureCompleteAndReturn(checkRequirementFutures.get(1));
            });
        }
    };
}

Example 13 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class FineGrainedSlotManagerTest method testNotificationAboutNotEnoughResources.

private void testNotificationAboutNotEnoughResources(boolean withNotificationGracePeriod) throws Exception {
    final JobID jobId = new JobID();
    final List<Tuple2<JobID, Collection<ResourceRequirement>>> notEnoughResourceNotifications = new ArrayList<>();
    final CompletableFuture<Void> notifyNotEnoughResourceFuture = new CompletableFuture<>();
    new Context() {

        {
            resourceActionsBuilder.setNotEnoughResourcesConsumer((jobId1, acquiredResources) -> {
                notEnoughResourceNotifications.add(Tuple2.of(jobId1, acquiredResources));
                notifyNotEnoughResourceFuture.complete(null);
            });
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction(((jobIDCollectionMap, taskManagerResourceInfoProvider) -> ResourceAllocationResult.builder().addUnfulfillableJob(jobId).build()));
            runTest(() -> {
                if (withNotificationGracePeriod) {
                    // this should disable notifications
                    runInMainThread(() -> getSlotManager().setFailUnfulfillableRequest(false));
                }
                final ResourceRequirements resourceRequirements = createResourceRequirements(jobId, 1);
                runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements));
                if (withNotificationGracePeriod) {
                    assertFutureNotComplete(notifyNotEnoughResourceFuture);
                    assertThat(notEnoughResourceNotifications, empty());
                    // re-enable notifications which should also trigger another
                    // resource check
                    runInMainThread(() -> getSlotManager().setFailUnfulfillableRequest(true));
                }
                assertFutureCompleteAndReturn(notifyNotEnoughResourceFuture);
                assertThat(notEnoughResourceNotifications, hasSize(1));
                final Tuple2<JobID, Collection<ResourceRequirement>> notification = notEnoughResourceNotifications.get(0);
                assertThat(notification.f0, is(jobId));
            });
        }
    };
}

Example 14 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManager method checkResourceRequirements.

// ---------------------------------------------------------------------------------------------
// Requirement matching
// ---------------------------------------------------------------------------------------------
/**
 * Matches resource requirements against available resources. In a first round requirements are
 * matched against free slot, and any match results in a slot allocation. The remaining
 * unfulfilled requirements are matched against pending slots, allocating more workers if no
 * matching pending slot could be found. If the requirements for a job could not be fulfilled
 * then a notification is sent to the job master informing it as such.
 *
 * <p>Performance notes: At it's core this method loops, for each job, over all free/pending
 * slots for each required slot, trying to find a matching slot. One should generally go in with
 * the assumption that this runs in numberOfJobsRequiringResources * numberOfRequiredSlots *
 * numberOfFreeOrPendingSlots. This is especially important when dealing with pending slots, as
 * matches between requirements and pending slots are not persisted and recomputed on each call.
 * This may required further refinements in the future; e.g., persisting the matches between
 * requirements and pending slots, or not matching against pending slots at all.
 *
 * <p>When dealing with unspecific resource profiles (i.e., {@link ResourceProfile#ANY}/{@link
 * ResourceProfile#UNKNOWN}), then the number of free/pending slots is not relevant because we
 * only need exactly 1 comparison to determine whether a slot can be fulfilled or not, since
 * they are all the same anyway.
 *
 * <p>When dealing with specific resource profiles things can be a lot worse, with the classical
 * cases where either no matches are found, or only at the very end of the iteration. In the
 * absolute worst case, with J jobs, requiring R slots each with a unique resource profile such
 * each pair of these profiles is not matching, and S free/pending slots that don't fulfill any
 * requirement, then this method does a total of J*R*S resource profile comparisons.
 */
private void checkResourceRequirements() {
    final Map<JobID, Collection<ResourceRequirement>> missingResources = resourceTracker.getMissingResources();
    if (missingResources.isEmpty()) {
        return;
    }
    final Map<JobID, ResourceCounter> unfulfilledRequirements = new LinkedHashMap<>();
    for (Map.Entry<JobID, Collection<ResourceRequirement>> resourceRequirements : missingResources.entrySet()) {
        final JobID jobId = resourceRequirements.getKey();
        final ResourceCounter unfulfilledJobRequirements = tryAllocateSlotsForJob(jobId, resourceRequirements.getValue());
        if (!unfulfilledJobRequirements.isEmpty()) {
            unfulfilledRequirements.put(jobId, unfulfilledJobRequirements);
        }
    }
    if (unfulfilledRequirements.isEmpty()) {
        return;
    }
    ResourceCounter pendingSlots = ResourceCounter.withResources(taskExecutorManager.getPendingTaskManagerSlots().stream().collect(Collectors.groupingBy(PendingTaskManagerSlot::getResourceProfile, Collectors.summingInt(x -> 1))));
    for (Map.Entry<JobID, ResourceCounter> unfulfilledRequirement : unfulfilledRequirements.entrySet()) {
        pendingSlots = tryFulfillRequirementsWithPendingSlots(unfulfilledRequirement.getKey(), unfulfilledRequirement.getValue().getResourcesWithCount(), pendingSlots);
    }
}

Example 15 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DefaultDeclareResourceRequirementServiceConnectionManagerTest method runStopSendingResourceRequirementsTest.

private void runStopSendingResourceRequirementsTest(Consumer<DeclareResourceRequirementServiceConnectionManager> testAction) throws InterruptedException {
    final DeclareResourceRequirementServiceConnectionManager declareResourceRequirementServiceConnectionManager = createResourceManagerConnectionManager();
    final FailingDeclareResourceRequirementsService declareResourceRequirementsService = new FailingDeclareResourceRequirementsService(1);
    declareResourceRequirementServiceConnectionManager.connect(declareResourceRequirementsService);
    final ResourceRequirements resourceRequirements = createResourceRequirements();
    declareResourceRequirementServiceConnectionManager.declareResourceRequirements(resourceRequirements);
    declareResourceRequirementsService.waitForResourceRequirementsDeclaration();
    testAction.accept(declareResourceRequirementServiceConnectionManager);
    scheduledExecutor.triggerNonPeriodicScheduledTasksWithRecursion();
    assertThat(declareResourceRequirementsService.hasResourceRequirements(), is(false));
}