Examples with TaskExecutorConnection org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection used on opensource projects

Example 36 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method registerTaskExecutorWithTwoSlots.

private void registerTaskExecutorWithTwoSlots(DeclarativeSlotManager slotManager, CompletableFuture<JobID> firstRequestSlotFuture) {
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple6 -> {
        firstRequestSlotFuture.complete(slotIDJobIDAllocationIDStringResourceManagerIdTuple6.f1);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection firstTaskExecutorConnection = createTaskExecutorConnection(taskExecutorGateway);
    final SlotReport firstSlotReport = createSlotReport(firstTaskExecutorConnection.getResourceID(), 2);
    slotManager.registerTaskManager(firstTaskExecutorConnection, firstSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
}

Example 37 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all registered slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection(taskExecutorGateway);
    final ResourceID resourceId = taskManagerConnection.getResourceID();
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createAllocatedSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertEquals("The number registered slots does not equal the expected number.", 2, slotManager.getNumberRegisteredSlots());
        slotManager.processResourceRequirements(resourceRequirements);
        slotManager.unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION);
        assertEquals(0, slotManager.getNumberRegisteredSlots());
    }
}

Example 38 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class TaskExecutorManagerTest method createAndRegisterTaskExecutor.

private static InstanceID createAndRegisterTaskExecutor(TaskExecutorManager taskExecutorManager, int numSlots, ResourceProfile resourceProfile) {
    final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection();
    List<SlotStatus> slotStatuses = IntStream.range(0, numSlots).mapToObj(slotNumber -> new SlotStatus(new SlotID(taskExecutorConnection.getResourceID(), slotNumber), resourceProfile)).collect(Collectors.toList());
    final SlotReport slotReport = new SlotReport(slotStatuses);
    taskExecutorManager.registerTaskManager(taskExecutorConnection, slotReport, resourceProfile.multiply(numSlots), resourceProfile);
    return taskExecutorConnection.getInstanceID();
}

Example 39 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class FineGrainedSlotManagerTest method testTaskManagerRegistrationDeductPendingTaskManager.

/**
 * Tests that we can matched task manager will deduct pending task manager.
 */
@Test
public void testTaskManagerRegistrationDeductPendingTaskManager() throws Exception {
    final TaskExecutorConnection taskExecutionConnection1 = createTaskExecutorConnection();
    final TaskExecutorConnection taskExecutionConnection2 = createTaskExecutorConnection();
    final TaskExecutorConnection taskExecutionConnection3 = createTaskExecutorConnection();
    final SlotReport slotReportWithAllocatedSlot = new SlotReport(createAllocatedSlotStatus(new AllocationID(), DEFAULT_SLOT_RESOURCE_PROFILE));
    new Context() {

        {
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture1 = new CompletableFuture<>();
                final CompletableFuture<Boolean> registerTaskManagerFuture2 = new CompletableFuture<>();
                final CompletableFuture<Boolean> registerTaskManagerFuture3 = new CompletableFuture<>();
                runInMainThread(() -> {
                    getTaskManagerTracker().addPendingTaskManager(new PendingTaskManager(DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_NUM_SLOTS_PER_WORKER));
                    // task manager with allocated slot cannot deduct pending
                    // task manager
                    registerTaskManagerFuture1.complete(getSlotManager().registerTaskManager(taskExecutionConnection1, slotReportWithAllocatedSlot, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                });
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture1), is(true));
                assertThat(getTaskManagerTracker().getPendingTaskManagers().size(), is(1));
                // task manager with mismatched resource cannot deduct
                // pending task manager
                runInMainThread(() -> registerTaskManagerFuture2.complete(getSlotManager().registerTaskManager(taskExecutionConnection2, new SlotReport(), LARGE_TOTAL_RESOURCE_PROFILE, LARGE_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture2), is(true));
                assertThat(getTaskManagerTracker().getPendingTaskManagers().size(), is(1));
                runInMainThread(() -> registerTaskManagerFuture3.complete(getSlotManager().registerTaskManager(taskExecutionConnection3, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture3), is(true));
                assertThat(getTaskManagerTracker().getPendingTaskManagers().size(), is(0));
            });
        }
    };
}

Example 40 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class FineGrainedSlotManagerTest method testSlotAllocationAccordingToStrategyResult.

// ---------------------------------------------------------------------------------------------
// Handle result from ResourceAllocationStrategy
// ---------------------------------------------------------------------------------------------
@Test
public void testSlotAllocationAccordingToStrategyResult() throws Exception {
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestSlotFuture = new CompletableFuture<>();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestSlotFuture.complete(tuple6);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final JobID jobId = new JobID();
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction(((jobIDCollectionMap, taskManagerResourceInfoProvider) -> ResourceAllocationResult.builder().addAllocationOnRegisteredResource(jobId, taskManagerConnection.getInstanceID(), DEFAULT_SLOT_RESOURCE_PROFILE).build()));
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                    getSlotManager().processResourceRequirements(createResourceRequirements(jobId, 1));
                });
                final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> requestSlot = assertFutureCompleteAndReturn(requestSlotFuture);
                assertEquals(jobId, requestSlot.f1);
                assertEquals(DEFAULT_SLOT_RESOURCE_PROFILE, requestSlot.f3);
            });
        }
    };
}