Examples with TaskExecutorConnection org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection used on opensource projects

Example 26 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
    final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        allocationIdFuture.complete(tuple6.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    // The fatal error handler will exit the system if there is any exceptions in handling the
    // ack of request slot. We need the security manager to verify that would not happen.
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().processResourceRequirements(createResourceRequirements(new JobID(), 1));
                    getSlotManager().registerTaskManager(taskExecutionConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                });
                final AllocationID allocationId = assertFutureCompleteAndReturn(allocationIdFuture);
                runInMainThread(() -> getSlotManager().reportSlotStatus(taskExecutionConnection.getInstanceID(), new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE))));
                assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
            });
        }
    };
    System.setSecurityManager(null);
}

Example 27 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob() throws Exception {
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    final CompletableFuture<AllocationID> firstSlotAllocationIdFuture = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> firstSlotRequestAcknowledgeFuture = new CompletableFuture<>();
    final Iterator<CompletableFuture<Acknowledge>> slotRequestAcknowledgeFutures = Arrays.asList(firstSlotRequestAcknowledgeFuture, new CompletableFuture<Acknowledge>()).iterator();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(requestSlotParameters -> {
        firstSlotAllocationIdFuture.complete(requestSlotParameters.f2);
        return slotRequestAcknowledgeFutures.next();
    }).createTestingTaskExecutorGateway();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        final SlotID slotId = Iterators.getOnlyElement(slotReport.iterator()).getSlotID();
        // register task executor
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReport(taskExecutionConnection.getResourceID(), 1));
        // triggers the allocation of a slot
        final JobID firstJobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(firstJobId, 1));
        // clear requirements immediately to ensure the slot will not get re-allocated to the
        // same job
        slotManager.processResourceRequirements(ResourceRequirements.empty(firstJobId, "foobar"));
        // when the slot is freed it will be re-assigned to this second job
        slotManager.processResourceRequirements(createResourceRequirements(new JobID(), 1));
        slotManager.freeSlot(slotId, firstSlotAllocationIdFuture.get());
        // acknowledge the first allocation
        // this should fail if the acknowledgement is not ignored
        firstSlotRequestAcknowledgeFuture.complete(Acknowledge.get());
        // sanity check that the acknowledge was really ignored
        assertThat(slotTracker.getSlot(slotId).getJobId(), is(not(firstJobId)));
    }
}

Example 28 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfTaskExecutorUnregistered.

@Test
public void testAllocationUpdatesIgnoredIfTaskExecutorUnregistered() throws Exception {
    final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
        JobID jobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.unregisterTaskManager(taskExecutionConnection.getInstanceID(), TEST_EXCEPTION);
        executor.triggerAll();
        assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
    } finally {
        System.setSecurityManager(null);
    }
}

Example 29 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testReclaimInactiveSlotsOnClearRequirements.

@Test
public void testReclaimInactiveSlotsOnClearRequirements() throws Exception {
    final CompletableFuture<JobID> freeInactiveSlotsJobIdFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeInactiveSlotsConsumer(freeInactiveSlotsJobIdFuture::complete).createTestingTaskExecutorGateway();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
        final JobID jobId = new JobID();
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        // setup initial requirements, which should not trigger slots being reclaimed
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 2));
        assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
        // set requirements to 0, which should not trigger slots being reclaimed
        slotManager.processResourceRequirements(ResourceRequirements.empty(jobId, "foobar"));
        assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
        // clear requirements, which should trigger slots being reclaimed
        slotManager.clearResourceRequirements(jobId);
        assertThat(freeInactiveSlotsJobIdFuture.get(), is(jobId));
    }
}

Example 30 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DeclarativeSlotManagerTest method testTaskExecutorSlotAllocationTimeoutHandling.

/**
 * Tests that a slot allocation is retried if it times out on the task manager side.
 */
@Test
public void testTaskExecutorSlotAllocationTimeoutHandling() throws Exception {
    final JobID jobId = new JobID();
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot(jobId);
    final CompletableFuture<Acknowledge> slotRequestFuture1 = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> slotRequestFuture2 = new CompletableFuture<>();
    final Iterator<CompletableFuture<Acknowledge>> slotRequestFutureIterator = Arrays.asList(slotRequestFuture1, slotRequestFuture2).iterator();
    final ArrayBlockingQueue<SlotID> slotIds = new ArrayBlockingQueue<>(2);
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(FunctionUtils.uncheckedFunction(requestSlotParameters -> {
        slotIds.put(requestSlotParameters.f0);
        return slotRequestFutureIterator.next();
    })).createTestingTaskExecutorGateway();
    final ResourceID resourceId = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createFreeSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
        slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(resourceRequirements);
        final SlotID firstSlotId = slotIds.take();
        assertThat(slotIds, is(empty()));
        DeclarativeTaskManagerSlot failedSlot = slotTracker.getSlot(firstSlotId);
        // let the first attempt fail --> this should trigger a second attempt
        slotRequestFuture1.completeExceptionally(new SlotAllocationException("Test exception."));
        assertThat(getTotalResourceCount(resourceTracker.getAcquiredResources(jobId)), is(1));
        // the second attempt succeeds
        slotRequestFuture2.complete(Acknowledge.get());
        final SlotID secondSlotId = slotIds.take();
        assertThat(slotIds, is(empty()));
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(secondSlotId);
        assertThat(slot.getState(), is(SlotState.ALLOCATED));
        assertEquals(jobId, slot.getJobId());
        if (!failedSlot.getSlotId().equals(slot.getSlotId())) {
            assertThat(failedSlot.getState(), is(SlotState.FREE));
        }
    }
}