use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManagerTest method testReclaimInactiveSlotsOnClearRequirements.
@Test
public void testReclaimInactiveSlotsOnClearRequirements() throws Exception {
final CompletableFuture<JobID> freeInactiveSlotsJobIdFuture = new CompletableFuture<>();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeInactiveSlotsConsumer(freeInactiveSlotsJobIdFuture::complete).createTestingTaskExecutorGateway();
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
final JobID jobId = new JobID();
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1);
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
// setup initial requirements, which should not trigger slots being reclaimed
slotManager.processResourceRequirements(createResourceRequirements(jobId, 2));
assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
// set requirements to 0, which should not trigger slots being reclaimed
slotManager.processResourceRequirements(ResourceRequirements.empty(jobId, "foobar"));
assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
// clear requirements, which should trigger slots being reclaimed
slotManager.clearResourceRequirements(jobId);
assertThat(freeInactiveSlotsJobIdFuture.get(), is(jobId));
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManagerTest method testTaskExecutorSlotAllocationTimeoutHandling.
/**
* Tests that a slot allocation is retried if it times out on the task manager side.
*/
@Test
public void testTaskExecutorSlotAllocationTimeoutHandling() throws Exception {
final JobID jobId = new JobID();
final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot(jobId);
final CompletableFuture<Acknowledge> slotRequestFuture1 = new CompletableFuture<>();
final CompletableFuture<Acknowledge> slotRequestFuture2 = new CompletableFuture<>();
final Iterator<CompletableFuture<Acknowledge>> slotRequestFutureIterator = Arrays.asList(slotRequestFuture1, slotRequestFuture2).iterator();
final ArrayBlockingQueue<SlotID> slotIds = new ArrayBlockingQueue<>(2);
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(FunctionUtils.uncheckedFunction(requestSlotParameters -> {
slotIds.put(requestSlotParameters.f0);
return slotRequestFutureIterator.next();
})).createTestingTaskExecutorGateway();
final ResourceID resourceId = ResourceID.generate();
final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);
final SlotID slotId1 = new SlotID(resourceId, 0);
final SlotID slotId2 = new SlotID(resourceId, 1);
final SlotReport slotReport = new SlotReport(Arrays.asList(createFreeSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
final ResourceTracker resourceTracker = new DefaultResourceTracker();
final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
slotManager.registerTaskManager(taskManagerConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
slotManager.processResourceRequirements(resourceRequirements);
final SlotID firstSlotId = slotIds.take();
assertThat(slotIds, is(empty()));
DeclarativeTaskManagerSlot failedSlot = slotTracker.getSlot(firstSlotId);
// let the first attempt fail --> this should trigger a second attempt
slotRequestFuture1.completeExceptionally(new SlotAllocationException("Test exception."));
assertThat(getTotalResourceCount(resourceTracker.getAcquiredResources(jobId)), is(1));
// the second attempt succeeds
slotRequestFuture2.complete(Acknowledge.get());
final SlotID secondSlotId = slotIds.take();
assertThat(slotIds, is(empty()));
DeclarativeTaskManagerSlot slot = slotTracker.getSlot(secondSlotId);
assertThat(slot.getState(), is(SlotState.ALLOCATED));
assertEquals(jobId, slot.getJobId());
if (!failedSlot.getSlotId().equals(slot.getSlotId())) {
assertThat(failedSlot.getState(), is(SlotState.FREE));
}
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManagerTest method testClearRequirementsClearsResourceTracker.
@Test
public void testClearRequirementsClearsResourceTracker() throws Exception {
final ResourceTracker resourceTracker = new DefaultResourceTracker();
final CompletableFuture<JobID> freeInactiveSlotsJobIdFuture = new CompletableFuture<>();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeInactiveSlotsConsumer(freeInactiveSlotsJobIdFuture::complete).createTestingTaskExecutorGateway();
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
final JobID jobId = new JobID();
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1);
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
slotManager.processResourceRequirements(createResourceRequirements(jobId, 2));
slotManager.clearResourceRequirements(jobId);
assertThat(resourceTracker.getMissingResources().keySet(), empty());
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManagerTest method testReportAllocatedSlot.
/**
* Tests that free slots which are reported as allocated won't be considered for fulfilling
* other pending slot requests.
*
* <p>See: FLINK-8505
*/
@Test
public void testReportAllocatedSlot() throws Exception {
final ResourceID taskManagerId = ResourceID.generate();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskManagerId, taskExecutorGateway);
final ResourceTracker resourceTracker = new DefaultResourceTracker();
final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).setSlotTracker(slotTracker).buildAndStartWithDirectExec()) {
// initially report a single slot as free
final SlotID slotId = new SlotID(taskManagerId, 0);
final SlotReport initialSlotReport = new SlotReport(createFreeSlotStatus(slotId));
slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport, ResourceProfile.ANY, ResourceProfile.ANY);
assertThat(slotManager.getNumberRegisteredSlots(), is(equalTo(1)));
// Now report this slot as allocated
final SlotStatus slotStatus = createAllocatedSlotStatus(slotId);
final SlotReport slotReport = new SlotReport(slotStatus);
slotManager.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport);
final JobID jobId = new JobID();
// this resource requirement should not be fulfilled
ResourceRequirements requirements = createResourceRequirementsForSingleSlot(jobId);
slotManager.processResourceRequirements(requirements);
assertThat(slotTracker.getSlot(slotId).getJobId(), is(slotStatus.getJobID()));
assertThat(getTotalResourceCount(resourceTracker.getMissingResources().get(jobId)), is(1));
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.
@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
final ResourceTracker resourceTracker = new DefaultResourceTracker();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
System.setSecurityManager(trackingSecurityManager);
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
JobID jobId = new JobID();
slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1));
executor.triggerAll();
assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
} finally {
System.setSecurityManager(null);
}
}
Aggregations