Examples with ResourceRequirements org.apache.flink.runtime.slots.ResourceRequirements used on opensource projects

Example 6 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testRequirementDeclaration.

private void testRequirementDeclaration(RequirementDeclarationScenario scenario) throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final ResourceID resourceID = ResourceID.generate();
    final JobID jobId = new JobID();
    final SlotID slotId = new SlotID(resourceID, 0);
    final String targetAddress = "localhost";
    final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    // accept an incoming slot request
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(Tuple6.of(tuple6.f0, tuple6.f1, tuple6.f2, tuple6.f3, tuple6.f4, tuple6.f5));
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
    final SlotReport slotReport = new SlotReport(slotStatus);
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec(resourceManagerId, new TestingResourceActionsBuilder().build())) {
        if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_BEFORE_REQUIREMENT_DECLARATION) {
            slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        }
        final ResourceRequirements requirements = ResourceRequirements.create(jobId, targetAddress, Collections.singleton(ResourceRequirement.create(resourceProfile, 1)));
        slotManager.processResourceRequirements(requirements);
        if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_AFTER_REQUIREMENT_DECLARATION) {
            slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        }
        assertThat(requestFuture.get(), is(equalTo(Tuple6.of(slotId, jobId, requestFuture.get().f2, resourceProfile, targetAddress, resourceManagerId))));
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertEquals("The slot has not been allocated to the expected allocation id.", jobId, slot.getJobId());
    }
}

Example 7 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testNotificationAboutNotEnoughResources.

private static void testNotificationAboutNotEnoughResources(boolean withNotificationGracePeriod) throws Exception {
    final JobID jobId = new JobID();
    final int numRequiredSlots = 3;
    final int numExistingSlots = 1;
    List<Tuple2<JobID, Collection<ResourceRequirement>>> notEnoughResourceNotifications = new ArrayList<>();
    ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceFunction(ignored -> false).setNotEnoughResourcesConsumer((jobId1, acquiredResources) -> notEnoughResourceNotifications.add(Tuple2.of(jobId1, acquiredResources))).build();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().buildAndStart(ResourceManagerId.generate(), new ManuallyTriggeredScheduledExecutor(), resourceManagerActions)) {
        if (withNotificationGracePeriod) {
            // this should disable notifications
            slotManager.setFailUnfulfillableRequest(false);
        }
        final ResourceID taskExecutorResourceId = ResourceID.generate();
        final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(taskExecutorResourceId, new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway());
        final SlotReport slotReport = createSlotReport(taskExecutorResourceId, numExistingSlots);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        ResourceRequirements resourceRequirements = createResourceRequirements(jobId, numRequiredSlots);
        slotManager.processResourceRequirements(resourceRequirements);
        if (withNotificationGracePeriod) {
            assertThat(notEnoughResourceNotifications, empty());
            // re-enable notifications which should also trigger another resource check
            slotManager.setFailUnfulfillableRequest(true);
        }
        assertThat(notEnoughResourceNotifications, hasSize(1));
        Tuple2<JobID, Collection<ResourceRequirement>> notification = notEnoughResourceNotifications.get(0);
        assertThat(notification.f0, is(jobId));
        assertThat(notification.f1, hasItem(ResourceRequirement.create(ResourceProfile.ANY, numExistingSlots)));
        // another slot report that does not indicate any changes should not trigger another
        // notification
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), slotReport);
        assertThat(notEnoughResourceNotifications, hasSize(1));
    }
}

Example 8 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotRequestFailure.

/**
 * Tests that the SlotManager retries allocating a slot if the TaskExecutor#requestSlot call
 * fails.
 */
@Test
public void testSlotRequestFailure() throws Exception {
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        ResourceRequirements requirements = createResourceRequirementsForSingleSlot();
        slotManager.processResourceRequirements(requirements);
        final BlockingQueue<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestSlotQueue = new ArrayBlockingQueue<>(1);
        final BlockingQueue<CompletableFuture<Acknowledge>> responseQueue = new ArrayBlockingQueue<>(2);
        final CompletableFuture<Acknowledge> firstManualSlotRequestResponse = new CompletableFuture<>();
        responseQueue.offer(firstManualSlotRequestResponse);
        final CompletableFuture<Acknowledge> secondManualSlotRequestResponse = new CompletableFuture<>();
        responseQueue.offer(secondManualSlotRequestResponse);
        final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple6 -> {
            requestSlotQueue.offer(slotIDJobIDAllocationIDStringResourceManagerIdTuple6);
            try {
                return responseQueue.take();
            } catch (InterruptedException ignored) {
                return FutureUtils.completedExceptionally(new FlinkException("Response queue was interrupted."));
            }
        }).createTestingTaskExecutorGateway();
        final ResourceID taskExecutorResourceId = ResourceID.generate();
        final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(taskExecutorResourceId, testingTaskExecutorGateway);
        final SlotReport slotReport = new SlotReport(createFreeSlotStatus(new SlotID(taskExecutorResourceId, 0)));
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> firstRequest = requestSlotQueue.take();
        // fail first request
        firstManualSlotRequestResponse.completeExceptionally(new SlotAllocationException("Test exception"));
        final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> secondRequest = requestSlotQueue.take();
        assertThat(secondRequest.f1, equalTo(firstRequest.f1));
        assertThat(secondRequest.f0, equalTo(firstRequest.f0));
        secondManualSlotRequestResponse.complete(Acknowledge.get());
        final DeclarativeTaskManagerSlot slot = slotTracker.getSlot(secondRequest.f0);
        assertThat(slot.getState(), equalTo(SlotState.ALLOCATED));
        assertThat(slot.getJobId(), equalTo(secondRequest.f1));
    }
}

Example 9 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testRequirementDeclarationWithResourceAllocationFailure.

/**
 * Tests that resources continue to be considered missing if we cannot allocate more resources.
 */
@Test
public void testRequirementDeclarationWithResourceAllocationFailure() throws Exception {
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceFunction(value -> false).build();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStartWithDirectExec(ResourceManagerId.generate(), resourceManagerActions)) {
        slotManager.processResourceRequirements(resourceRequirements);
        final JobID jobId = resourceRequirements.getJobId();
        assertThat(getTotalResourceCount(resourceTracker.getMissingResources().get(jobId)), is(1));
    }
}

Example 10 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testDuplicateResourceRequirementDeclarationAfterSuccessfulAllocation.

/**
 * Tests that duplicate resource requirement declaration do not result in additional slots being
 * allocated after a pending slot request has been fulfilled but not yet freed.
 */
@Test
public void testDuplicateResourceRequirementDeclarationAfterSuccessfulAllocation() throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final AtomicInteger allocateResourceCalls = new AtomicInteger(0);
    final ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceConsumer(ignored -> allocateResourceCalls.incrementAndGet()).build();
    ResourceRequirements requirements = createResourceRequirementsForSingleSlot();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    final ResourceID resourceID = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    final SlotID slotId = new SlotID(resourceID, 0);
    final SlotReport slotReport = new SlotReport(createFreeSlotStatus(slotId));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec(resourceManagerId, resourceManagerActions)) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(requirements);
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertThat(slot.getState(), is(SlotState.ALLOCATED));
        slotManager.processResourceRequirements(requirements);
    }
    // check that we have only called the resource allocation only for the first slot request,
    // since the second request is a duplicate
    assertThat(allocateResourceCalls.get(), is(0));
}