Examples with TaskExecutorConnection org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection used on opensource projects

Example 6 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testUpdateSlotReport.

/**
 * Tests that slots are updated with respect to the latest incoming slot report. This means that
 * slots for which a report was received are updated accordingly.
 */
@Test
public void testUpdateSlotReport() throws Exception {
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection();
    final ResourceID resourceId = taskManagerConnection.getResourceID();
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotStatus slotStatus1 = createFreeSlotStatus(slotId1);
    final SlotStatus slotStatus2 = createFreeSlotStatus(slotId2);
    final SlotStatus newSlotStatus2 = createAllocatedSlotStatus(slotId2);
    final JobID jobId = newSlotStatus2.getJobID();
    final SlotReport slotReport1 = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
    final SlotReport slotReport2 = new SlotReport(Arrays.asList(newSlotStatus2, slotStatus1));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        // check that we don't have any slots registered
        assertEquals(0, slotManager.getNumberRegisteredSlots());
        slotManager.registerTaskManager(taskManagerConnection, slotReport1, ResourceProfile.ANY, ResourceProfile.ANY);
        DeclarativeTaskManagerSlot slot1 = slotTracker.getSlot(slotId1);
        DeclarativeTaskManagerSlot slot2 = slotTracker.getSlot(slotId2);
        assertEquals(2, slotManager.getNumberRegisteredSlots());
        assertSame(SlotState.FREE, slot1.getState());
        assertSame(SlotState.FREE, slot2.getState());
        assertTrue(slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), slotReport2));
        assertEquals(2, slotManager.getNumberRegisteredSlots());
        assertNotNull(slotTracker.getSlot(slotId1));
        assertNotNull(slotTracker.getSlot(slotId2));
        // slot1 should still be free, slot2 should have been allocated
        assertSame(SlotState.FREE, slot1.getState());
        assertEquals(jobId, slotTracker.getSlot(slotId2).getJobId());
    }
}

Example 7 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testFreeSlot.

/**
 * Tests that freeing a slot will correctly reset the slot and mark it as a free slot.
 */
@Test
public void testFreeSlot() throws Exception {
    final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection();
    final ResourceID resourceID = taskExecutorConnection.getResourceID();
    final SlotID slotId = new SlotID(resourceID, 0);
    final SlotReport slotReport = new SlotReport(createAllocatedSlotStatus(slotId));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertSame(SlotState.ALLOCATED, slot.getState());
        slotManager.freeSlot(slotId, new AllocationID());
        assertSame(SlotState.FREE, slot.getState());
        assertEquals(1, slotManager.getNumberFreeSlots());
    }
}

Example 8 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskManagerRegistration.

/**
 * Tests that we can register task manager and their slots at the slot manager.
 */
@Test
public void testTaskManagerRegistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    final ResourceID resourceId = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createFreeSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertThat("The number registered slots does not equal the expected number.", slotManager.getNumberRegisteredSlots(), is(2));
        assertNotNull(slotTracker.getSlot(slotId1));
        assertNotNull(slotTracker.getSlot(slotId2));
    }
}

Example 9 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotRequestFailure.

/**
 * Tests that the SlotManager retries allocating a slot if the TaskExecutor#requestSlot call
 * fails.
 */
@Test
public void testSlotRequestFailure() throws Exception {
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        ResourceRequirements requirements = createResourceRequirementsForSingleSlot();
        slotManager.processResourceRequirements(requirements);
        final BlockingQueue<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestSlotQueue = new ArrayBlockingQueue<>(1);
        final BlockingQueue<CompletableFuture<Acknowledge>> responseQueue = new ArrayBlockingQueue<>(2);
        final CompletableFuture<Acknowledge> firstManualSlotRequestResponse = new CompletableFuture<>();
        responseQueue.offer(firstManualSlotRequestResponse);
        final CompletableFuture<Acknowledge> secondManualSlotRequestResponse = new CompletableFuture<>();
        responseQueue.offer(secondManualSlotRequestResponse);
        final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple6 -> {
            requestSlotQueue.offer(slotIDJobIDAllocationIDStringResourceManagerIdTuple6);
            try {
                return responseQueue.take();
            } catch (InterruptedException ignored) {
                return FutureUtils.completedExceptionally(new FlinkException("Response queue was interrupted."));
            }
        }).createTestingTaskExecutorGateway();
        final ResourceID taskExecutorResourceId = ResourceID.generate();
        final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(taskExecutorResourceId, testingTaskExecutorGateway);
        final SlotReport slotReport = new SlotReport(createFreeSlotStatus(new SlotID(taskExecutorResourceId, 0)));
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> firstRequest = requestSlotQueue.take();
        // fail first request
        firstManualSlotRequestResponse.completeExceptionally(new SlotAllocationException("Test exception"));
        final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> secondRequest = requestSlotQueue.take();
        assertThat(secondRequest.f1, equalTo(firstRequest.f1));
        assertThat(secondRequest.f0, equalTo(firstRequest.f0));
        secondManualSlotRequestResponse.complete(Acknowledge.get());
        final DeclarativeTaskManagerSlot slot = slotTracker.getSlot(secondRequest.f0);
        assertThat(slot.getState(), equalTo(SlotState.ALLOCATED));
        assertThat(slot.getJobId(), equalTo(secondRequest.f1));
    }
}

Example 10 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testDuplicateResourceRequirementDeclarationAfterSuccessfulAllocation.

/**
 * Tests that duplicate resource requirement declaration do not result in additional slots being
 * allocated after a pending slot request has been fulfilled but not yet freed.
 */
@Test
public void testDuplicateResourceRequirementDeclarationAfterSuccessfulAllocation() throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final AtomicInteger allocateResourceCalls = new AtomicInteger(0);
    final ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceConsumer(ignored -> allocateResourceCalls.incrementAndGet()).build();
    ResourceRequirements requirements = createResourceRequirementsForSingleSlot();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    final ResourceID resourceID = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    final SlotID slotId = new SlotID(resourceID, 0);
    final SlotReport slotReport = new SlotReport(createFreeSlotStatus(slotId));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec(resourceManagerId, resourceManagerActions)) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(requirements);
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertThat(slot.getState(), is(SlotState.ALLOCATED));
        slotManager.processResourceRequirements(requirements);
    }
    // check that we have only called the resource allocation only for the first slot request,
    // since the second request is a duplicate
    assertThat(allocateResourceCalls.get(), is(0));
}