Examples with TaskExecutorGateway org.apache.flink.runtime.taskexecutor.TaskExecutorGateway used on opensource projects

Example 41 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
    final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
        JobID jobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1));
        executor.triggerAll();
        assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
    } finally {
        System.setSecurityManager(null);
    }
}

Example 42 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method registerTaskExecutorWithTwoSlots.

private void registerTaskExecutorWithTwoSlots(DeclarativeSlotManager slotManager, CompletableFuture<JobID> firstRequestSlotFuture) {
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple6 -> {
        firstRequestSlotFuture.complete(slotIDJobIDAllocationIDStringResourceManagerIdTuple6.f1);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection firstTaskExecutorConnection = createTaskExecutorConnection(taskExecutorGateway);
    final SlotReport firstSlotReport = createSlotReport(firstTaskExecutorConnection.getResourceID(), 2);
    slotManager.registerTaskManager(firstTaskExecutorConnection, firstSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
}

Example 43 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all registered slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection(taskExecutorGateway);
    final ResourceID resourceId = taskManagerConnection.getResourceID();
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createAllocatedSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertEquals("The number registered slots does not equal the expected number.", 2, slotManager.getNumberRegisteredSlots());
        slotManager.processResourceRequirements(resourceRequirements);
        slotManager.unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION);
        assertEquals(0, slotManager.getNumberRegisteredSlots());
    }
}

Example 44 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class FineGrainedSlotManagerTest method testSlotAllocationAccordingToStrategyResult.

// ---------------------------------------------------------------------------------------------
// Handle result from ResourceAllocationStrategy
// ---------------------------------------------------------------------------------------------
@Test
public void testSlotAllocationAccordingToStrategyResult() throws Exception {
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestSlotFuture = new CompletableFuture<>();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestSlotFuture.complete(tuple6);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final JobID jobId = new JobID();
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction(((jobIDCollectionMap, taskManagerResourceInfoProvider) -> ResourceAllocationResult.builder().addAllocationOnRegisteredResource(jobId, taskManagerConnection.getInstanceID(), DEFAULT_SLOT_RESOURCE_PROFILE).build()));
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                    getSlotManager().processResourceRequirements(createResourceRequirements(jobId, 1));
                });
                final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> requestSlot = assertFutureCompleteAndReturn(requestSlotFuture);
                assertEquals(jobId, requestSlot.f1);
                assertEquals(DEFAULT_SLOT_RESOURCE_PROFILE, requestSlot.f3);
            });
        }
    };
}

Example 45 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class FineGrainedSlotManagerTest method testSlotAllocationForPendingTaskManagerWillBeRespected.

@Test
public void testSlotAllocationForPendingTaskManagerWillBeRespected() throws Exception {
    final JobID jobId = new JobID();
    final CompletableFuture<Void> requestResourceFuture = new CompletableFuture<>();
    final PendingTaskManager pendingTaskManager = new PendingTaskManager(DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_NUM_SLOTS_PER_WORKER);
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestSlotFuture = new CompletableFuture<>();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestSlotFuture.complete(tuple6);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    new Context() {

        {
            resourceAllocationStrategyBuilder.setTryFulfillRequirementsFunction(((jobIDCollectionMap, taskManagerResourceInfoProvider) -> ResourceAllocationResult.builder().addPendingTaskManagerAllocate(pendingTaskManager).addAllocationOnPendingResource(jobId, pendingTaskManager.getPendingTaskManagerId(), DEFAULT_SLOT_RESOURCE_PROFILE).build()));
            resourceActionsBuilder.setAllocateResourceConsumer(ignored -> requestResourceFuture.complete(null));
            runTest(() -> {
                runInMainThread(() -> getSlotManager().processResourceRequirements(createResourceRequirements(jobId, 1)));
                assertFutureCompleteAndReturn(requestResourceFuture);
                runInMainThread(() -> getSlotManager().registerTaskManager(taskManagerConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                final Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId> requestSlot = assertFutureCompleteAndReturn(requestSlotFuture);
                assertEquals(jobId, requestSlot.f1);
                assertEquals(DEFAULT_SLOT_RESOURCE_PROFILE, requestSlot.f3);
            });
        }
    };
}