use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.
the class AbstractFineGrainedSlotManagerITCase method testResourceCanBeAllocatedForDifferentJobAfterFree.
/**
* Tests that a resource allocated for one job can be allocated for another job after being
* freed.
*/
private void testResourceCanBeAllocatedForDifferentJobAfterFree(SecondRequirementDeclarationTime secondRequirementDeclarationTime) throws Exception {
final CompletableFuture<AllocationID> allocationIdFuture1 = new CompletableFuture<>();
final CompletableFuture<AllocationID> allocationIdFuture2 = new CompletableFuture<>();
final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
if (!allocationIdFuture1.isDone()) {
allocationIdFuture1.complete(tuple6.f2);
} else {
allocationIdFuture2.complete(tuple6.f2);
}
return CompletableFuture.completedFuture(Acknowledge.get());
}).createTestingTaskExecutorGateway();
final ResourceID resourceID = ResourceID.generate();
final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
final SlotReport slotReport = new SlotReport();
new Context() {
{
runTest(() -> {
runInMainThread(() -> {
getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_SLOT_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
getSlotManager().processResourceRequirements(resourceRequirements1);
});
final AllocationID allocationId1 = assertFutureCompleteAndReturn(allocationIdFuture1);
TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId1).get();
assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements1.getJobId(), slot.getJobId());
if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.BEFORE_FREE) {
runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
}
// clear resource requirements first so that the freed slot isn't
// immediately re-assigned to the job
runInMainThread(() -> {
getSlotManager().processResourceRequirements(ResourceRequirements.create(resourceRequirements1.getJobId(), resourceRequirements1.getTargetAddress(), Collections.emptyList()));
getSlotManager().freeSlot(SlotID.getDynamicSlotID(resourceID), allocationId1);
});
if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.AFTER_FREE) {
runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
}
slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(allocationIdFuture2)).get();
assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements2.getJobId(), slot.getJobId());
});
}
};
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.
the class AbstractFineGrainedSlotManagerITCase method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.
@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
allocationIdFuture.complete(tuple6.f2);
return CompletableFuture.completedFuture(Acknowledge.get());
}).createTestingTaskExecutorGateway();
// The fatal error handler will exit the system if there is any exceptions in handling the
// ack of request slot. We need the security manager to verify that would not happen.
final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
System.setSecurityManager(trackingSecurityManager);
final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
final SlotReport slotReport = new SlotReport();
new Context() {
{
runTest(() -> {
runInMainThread(() -> {
getSlotManager().processResourceRequirements(createResourceRequirements(new JobID(), 1));
getSlotManager().registerTaskManager(taskExecutionConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
});
final AllocationID allocationId = assertFutureCompleteAndReturn(allocationIdFuture);
runInMainThread(() -> getSlotManager().reportSlotStatus(taskExecutionConnection.getInstanceID(), new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE))));
assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
});
}
};
System.setSecurityManager(null);
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.
the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob.
@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob() throws Exception {
final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
final CompletableFuture<AllocationID> firstSlotAllocationIdFuture = new CompletableFuture<>();
final CompletableFuture<Acknowledge> firstSlotRequestAcknowledgeFuture = new CompletableFuture<>();
final Iterator<CompletableFuture<Acknowledge>> slotRequestAcknowledgeFutures = Arrays.asList(firstSlotRequestAcknowledgeFuture, new CompletableFuture<Acknowledge>()).iterator();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(requestSlotParameters -> {
firstSlotAllocationIdFuture.complete(requestSlotParameters.f2);
return slotRequestAcknowledgeFutures.next();
}).createTestingTaskExecutorGateway();
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
final SlotID slotId = Iterators.getOnlyElement(slotReport.iterator()).getSlotID();
// register task executor
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReport(taskExecutionConnection.getResourceID(), 1));
// triggers the allocation of a slot
final JobID firstJobId = new JobID();
slotManager.processResourceRequirements(createResourceRequirements(firstJobId, 1));
// clear requirements immediately to ensure the slot will not get re-allocated to the
// same job
slotManager.processResourceRequirements(ResourceRequirements.empty(firstJobId, "foobar"));
// when the slot is freed it will be re-assigned to this second job
slotManager.processResourceRequirements(createResourceRequirements(new JobID(), 1));
slotManager.freeSlot(slotId, firstSlotAllocationIdFuture.get());
// acknowledge the first allocation
// this should fail if the acknowledgement is not ignored
firstSlotRequestAcknowledgeFuture.complete(Acknowledge.get());
// sanity check that the acknowledge was really ignored
assertThat(slotTracker.getSlot(slotId).getJobId(), is(not(firstJobId)));
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.
the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfTaskExecutorUnregistered.
@Test
public void testAllocationUpdatesIgnoredIfTaskExecutorUnregistered() throws Exception {
final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
final ResourceTracker resourceTracker = new DefaultResourceTracker();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
System.setSecurityManager(trackingSecurityManager);
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
JobID jobId = new JobID();
slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
slotManager.unregisterTaskManager(taskExecutionConnection.getInstanceID(), TEST_EXCEPTION);
executor.triggerAll();
assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
} finally {
System.setSecurityManager(null);
}
}
use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.
the class DeclarativeSlotManagerTest method testReclaimInactiveSlotsOnClearRequirements.
@Test
public void testReclaimInactiveSlotsOnClearRequirements() throws Exception {
final CompletableFuture<JobID> freeInactiveSlotsJobIdFuture = new CompletableFuture<>();
final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeInactiveSlotsConsumer(freeInactiveSlotsJobIdFuture::complete).createTestingTaskExecutorGateway();
try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
final JobID jobId = new JobID();
final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
final SlotReport slotReport = createSlotReportWithAllocatedSlots(taskExecutionConnection.getResourceID(), jobId, 1);
slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
// setup initial requirements, which should not trigger slots being reclaimed
slotManager.processResourceRequirements(createResourceRequirements(jobId, 2));
assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
// set requirements to 0, which should not trigger slots being reclaimed
slotManager.processResourceRequirements(ResourceRequirements.empty(jobId, "foobar"));
assertThat(freeInactiveSlotsJobIdFuture.isDone(), is(false));
// clear requirements, which should trigger slots being reclaimed
slotManager.clearResourceRequirements(jobId);
assertThat(freeInactiveSlotsJobIdFuture.get(), is(jobId));
}
}
Aggregations