Examples with ResourceRequirements org.apache.flink.runtime.slots.ResourceRequirements used on opensource projects

Example 21 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskExecutorSlotAllocationTimeoutHandling.

/**
 * Tests that a slot allocation is retried if it times out on the task manager side.
 */
@Test
public void testTaskExecutorSlotAllocationTimeoutHandling() throws Exception {
    final JobID jobId = new JobID();
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot(jobId);
    final CompletableFuture<Acknowledge> slotRequestFuture1 = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> slotRequestFuture2 = new CompletableFuture<>();
    final Iterator<CompletableFuture<Acknowledge>> slotRequestFutureIterator = Arrays.asList(slotRequestFuture1, slotRequestFuture2).iterator();
    final ArrayBlockingQueue<SlotID> slotIds = new ArrayBlockingQueue<>(2);
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(FunctionUtils.uncheckedFunction(requestSlotParameters -> {
        slotIds.put(requestSlotParameters.f0);
        return slotRequestFutureIterator.next();
    })).createTestingTaskExecutorGateway();
    final ResourceID resourceId = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createFreeSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(resourceRequirements);
        final SlotID firstSlotId = slotIds.take();
        assertThat(slotIds, is(empty()));
        DeclarativeTaskManagerSlot failedSlot = slotTracker.getSlot(firstSlotId);
        // let the first attempt fail --> this should trigger a second attempt
        slotRequestFuture1.completeExceptionally(new SlotAllocationException("Test exception."));
        assertThat(getTotalResourceCount(resourceTracker.getAcquiredResources(jobId)), is(1));
        // the second attempt succeeds
        slotRequestFuture2.complete(Acknowledge.get());
        final SlotID secondSlotId = slotIds.take();
        assertThat(slotIds, is(empty()));
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(secondSlotId);
        assertThat(slot.getState(), is(SlotState.ALLOCATED));
        assertEquals(jobId, slot.getJobId());
        if (!failedSlot.getSlotId().equals(slot.getSlotId())) {
            assertThat(failedSlot.getState(), is(SlotState.FREE));
        }
    }
}

Example 22 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testReportAllocatedSlot.

/**
 * Tests that free slots which are reported as allocated won't be considered for fulfilling
 * other pending slot requests.
 *
 * <p>See: FLINK-8505
 */
@Test
public void testReportAllocatedSlot() throws Exception {
    final ResourceID taskManagerId = ResourceID.generate();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskManagerId, taskExecutorGateway);
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        // initially report a single slot as free
        final SlotID slotId = new SlotID(taskManagerId, 0);
        final SlotReport initialSlotReport = new SlotReport(createFreeSlotStatus(slotId));
        slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertThat(slotManager.getNumberRegisteredSlots(), is(equalTo(1)));
        // Now report this slot as allocated
        final SlotStatus slotStatus = createAllocatedSlotStatus(slotId);
        final SlotReport slotReport = new SlotReport(slotStatus);
        slotManager.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport);
        final JobID jobId = new JobID();
        // this resource requirement should not be fulfilled
        ResourceRequirements requirements = createResourceRequirementsForSingleSlot(jobId);
        slotManager.processResourceRequirements(requirements);
        assertThat(slotTracker.getSlot(slotId).getJobId(), is(slotStatus.getJobID()));
        assertThat(getTotalResourceCount(resourceTracker.getMissingResources().get(jobId)), is(1));
    }
}

Example 23 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation.

/**
 * Tests that a slot request with no free slots will trigger the resource allocation.
 */
@Test
public void testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation() throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    CompletableFuture<WorkerResourceSpec> allocateResourceFuture = new CompletableFuture<>();
    ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceConsumer(allocateResourceFuture::complete).build();
    try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {
        slotManager.processResourceRequirements(resourceRequirements);
        allocateResourceFuture.get();
    }
}

Example 24 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotCanBeAllocatedForDifferentJobAfterFree.

private void testSlotCanBeAllocatedForDifferentJobAfterFree(SecondRequirementDeclarationTime secondRequirementDeclarationTime) throws Exception {
    final AllocationID allocationId = new AllocationID();
    final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
    final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection();
    final ResourceID resourceID = taskManagerConnection.getResourceID();
    final SlotID slotId = new SlotID(resourceID, 0);
    final SlotReport slotReport = new SlotReport(createFreeSlotStatus(slotId));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(resourceRequirements1);
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements1.getJobId(), slot.getJobId());
        if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.BEFORE_FREE) {
            slotManager.processResourceRequirements(resourceRequirements2);
        }
        // clear resource requirements first so that the freed slot isn't immediately
        // re-assigned to the job
        slotManager.processResourceRequirements(ResourceRequirements.create(resourceRequirements1.getJobId(), resourceRequirements1.getTargetAddress(), Collections.emptyList()));
        slotManager.freeSlot(slotId, allocationId);
        if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.AFTER_FREE) {
            slotManager.processResourceRequirements(resourceRequirements2);
        }
        assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements2.getJobId(), slot.getJobId());
    }
}

Example 25 with ResourceRequirements

use of org.apache.flink.runtime.slots.ResourceRequirements in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all registered slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection(taskExecutorGateway);
    final ResourceID resourceId = taskManagerConnection.getResourceID();
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createAllocatedSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertEquals("The number registered slots does not equal the expected number.", 2, slotManager.getNumberRegisteredSlots());
        slotManager.processResourceRequirements(resourceRequirements);
        slotManager.unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION);
        assertEquals(0, slotManager.getNumberRegisteredSlots());
    }
}