Examples with TaskExecutorConnection org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection used on opensource projects

Example 31 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotAllocationTimeout.

/**
 * Tests that if a slot allocation times out we try to allocate another slot.
 */
@Test
public void testSlotAllocationTimeout() throws Exception {
    final CompletableFuture<Void> secondSlotRequestFuture = new CompletableFuture<>();
    final BlockingQueue<Supplier<CompletableFuture<Acknowledge>>> responseQueue = new ArrayBlockingQueue<>(2);
    responseQueue.add(() -> FutureUtils.completedExceptionally(new TimeoutException("timeout")));
    responseQueue.add(() -> {
        secondSlotRequestFuture.complete(null);
        return new CompletableFuture<>();
    });
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection(new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> responseQueue.remove().get()).createTestingTaskExecutorGateway());
    final SlotReport slotReport = createSlotReport(taskManagerConnection.getResourceID(), 2);
    final Executor mainThreadExecutor = TestingUtils.defaultExecutor();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().build()) {
        slotManager.start(ResourceManagerId.generate(), mainThreadExecutor, new TestingResourceActionsBuilder().build());
        CompletableFuture.runAsync(() -> slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY), mainThreadExecutor).thenRun(() -> slotManager.processResourceRequirements(createResourceRequirementsForSingleSlot())).get(5, TimeUnit.SECONDS);
        // a second request is only sent if the first request timed out
        secondSlotRequestFuture.get();
    }
}

Example 32 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testClearRequirementsClearsResourceTracker.

@Test
public void testClearRequirementsClearsResourceTracker() throws Exception {
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final CompletableFuture<JobID> freeInactiveSlotsJobIdFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeInactiveSlotsConsumer(freeInactiveSlotsJobIdFuture::complete).createTestingTaskExecutorGateway();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
        final JobID jobId = new JobID();
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 2));
        slotManager.clearResourceRequirements(jobId);
        assertThat(resourceTracker.getMissingResources().keySet(), empty());
    }
}

Example 33 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testReportAllocatedSlot.

/**
 * Tests that free slots which are reported as allocated won't be considered for fulfilling
 * other pending slot requests.
 *
 * <p>See: FLINK-8505
 */
@Test
public void testReportAllocatedSlot() throws Exception {
    final ResourceID taskManagerId = ResourceID.generate();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskManagerId, taskExecutorGateway);
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        // initially report a single slot as free
        final SlotID slotId = new SlotID(taskManagerId, 0);
        final SlotReport initialSlotReport = new SlotReport(createFreeSlotStatus(slotId));
        slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        assertThat(slotManager.getNumberRegisteredSlots(), is(equalTo(1)));
        // Now report this slot as allocated
        final SlotStatus slotStatus = createAllocatedSlotStatus(slotId);
        final SlotReport slotReport = new SlotReport(slotStatus);
        slotManager.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport);
        final JobID jobId = new JobID();
        // this resource requirement should not be fulfilled
        ResourceRequirements requirements = createResourceRequirementsForSingleSlot(jobId);
        slotManager.processResourceRequirements(requirements);
        assertThat(slotTracker.getSlot(slotId).getJobId(), is(slotStatus.getJobID()));
        assertThat(getTotalResourceCount(resourceTracker.getMissingResources().get(jobId)), is(1));
    }
}

Example 34 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotCanBeAllocatedForDifferentJobAfterFree.

private void testSlotCanBeAllocatedForDifferentJobAfterFree(SecondRequirementDeclarationTime secondRequirementDeclarationTime) throws Exception {
    final AllocationID allocationId = new AllocationID();
    final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
    final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
    final TaskExecutorConnection taskManagerConnection = createTaskExecutorConnection();
    final ResourceID resourceID = taskManagerConnection.getResourceID();
    final SlotID slotId = new SlotID(resourceID, 0);
    final SlotReport slotReport = new SlotReport(createFreeSlotStatus(slotId));
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(resourceRequirements1);
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements1.getJobId(), slot.getJobId());
        if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.BEFORE_FREE) {
            slotManager.processResourceRequirements(resourceRequirements2);
        }
        // clear resource requirements first so that the freed slot isn't immediately
        // re-assigned to the job
        slotManager.processResourceRequirements(ResourceRequirements.create(resourceRequirements1.getJobId(), resourceRequirements1.getTargetAddress(), Collections.emptyList()));
        slotManager.freeSlot(slotId, allocationId);
        if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.AFTER_FREE) {
            slotManager.processResourceRequirements(resourceRequirements2);
        }
        assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements2.getJobId(), slot.getJobId());
    }
}

Example 35 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
    final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
        JobID jobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1));
        executor.triggerAll();
        assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
    } finally {
        System.setSecurityManager(null);
    }
}