use of org.apache.flink.runtime.taskexecutor.slot.ThreadSafeTaskSlotTable in project flink by apache.
the class TaskExecutorTest method testFreeingInactiveSlotDoesNotFail.
/**
* Tests that freeing an inactive slot is a legal operation that does not throw an exception.
*/
@Test
public void testFreeingInactiveSlotDoesNotFail() throws Exception {
final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(new InstanceID(), taskExecutorIsRegistered, availableSlotFuture);
rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
final MultiShotLatch offerSlotsLatch = new MultiShotLatch();
final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
offerSlotsLatch.trigger();
return new CompletableFuture<>();
}).build();
rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);
final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();
final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation).setTaskSlotTable(taskSlotTable).setTaskStateManager(localStateStoresManager).build();
final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
final ThreadSafeTaskSlotTable<Task> threadSafeTaskSlotTable = new ThreadSafeTaskSlotTable<>(taskSlotTable, taskExecutor.getMainThreadExecutableForTesting());
try {
taskExecutor.start();
taskExecutor.waitUntilStarted();
final TaskExecutorGateway tmGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
taskExecutorIsRegistered.await();
jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
final AllocationID allocationId = new AllocationID();
requestSlot(tmGateway, jobId, allocationId, buildSlotID(0), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
offerSlotsLatch.await();
tmGateway.freeSlot(allocationId, new RuntimeException("test exception"), timeout).get();
assertThat(availableSlotFuture.get().f2, is(allocationId));
assertThat(threadSafeTaskSlotTable.getAllocationIdsPerJob(jobId), empty());
} finally {
RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
}
}
use of org.apache.flink.runtime.taskexecutor.slot.ThreadSafeTaskSlotTable in project flink by apache.
the class TaskExecutorTest method testSlotOfferResponseWithPendingSlotOffer.
/**
* Tests the behavior of the task executor when a slot offer response is received while a newer
* slot offer is in progress.
*/
private void testSlotOfferResponseWithPendingSlotOffer(final ResponseOrder responseOrder) throws Exception {
final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(new InstanceID(), taskExecutorIsRegistered, new CompletableFuture<>());
final CompletableFuture<Collection<SlotOffer>> firstOfferResponseFuture = new CompletableFuture<>();
final CompletableFuture<Collection<SlotOffer>> secondOfferResponseFuture = new CompletableFuture<>();
final Queue<CompletableFuture<Collection<SlotOffer>>> slotOfferResponses = new ArrayDeque<>(Arrays.asList(firstOfferResponseFuture, secondOfferResponseFuture));
final MultiShotLatch offerSlotsLatch = new MultiShotLatch();
final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
offerSlotsLatch.trigger();
return slotOfferResponses.remove();
}).build();
rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
final ThreadSafeTaskSlotTable<Task> threadSafeTaskSlotTable = new ThreadSafeTaskSlotTable<>(taskSlotTable, taskExecutor.getMainThreadExecutableForTesting());
final SlotOffer slotOffer1 = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
final SlotOffer slotOffer2 = new SlotOffer(new AllocationID(), 1, ResourceProfile.ANY);
try {
taskExecutor.start();
taskExecutor.waitUntilStarted();
final TaskExecutorGateway tmGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
// wait until task executor registered at the RM
taskExecutorIsRegistered.await();
// notify job leader to start slot offering
jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
// request the first slot
requestSlot(tmGateway, jobId, slotOffer1.getAllocationId(), buildSlotID(slotOffer1.getSlotIndex()), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
// wait until first slot offer as arrived
offerSlotsLatch.await();
// request second slot, triggering another offer containing both slots
int slotIndex = slotOffer2.getSlotIndex();
requestSlot(tmGateway, jobId, slotOffer2.getAllocationId(), buildSlotID(slotIndex), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
// wait until second slot offer as arrived
offerSlotsLatch.await();
switch(responseOrder) {
case ACCEPT_THEN_REJECT:
// accept the first offer, but reject both slots for the second offer
firstOfferResponseFuture.complete(Collections.singletonList(slotOffer1));
assertThat(threadSafeTaskSlotTable.getActiveTaskSlotAllocationIdsPerJob(jobId), empty());
secondOfferResponseFuture.complete(Collections.emptyList());
assertThat(threadSafeTaskSlotTable.getAllocationIdsPerJob(jobId), empty());
return;
case REJECT_THEN_ACCEPT:
// fail the first offer, but accept both slots for the second offer
// in the past the rejection of the first offer freed the slot; when the slot is
// accepted from the second offer the activation of said slot then failed
firstOfferResponseFuture.complete(Collections.emptyList());
secondOfferResponseFuture.complete(Arrays.asList(slotOffer1, slotOffer2));
assertThat(threadSafeTaskSlotTable.getAllocationIdsPerJob(jobId), containsInAnyOrder(slotOffer1.getAllocationId(), slotOffer2.getAllocationId()));
return;
}
} finally {
RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
}
}
use of org.apache.flink.runtime.taskexecutor.slot.ThreadSafeTaskSlotTable in project flink by apache.
the class TaskExecutorTest method testSlotOfferCounterIsSeparatedByJob.
@Test
public void testSlotOfferCounterIsSeparatedByJob() throws Exception {
final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(new InstanceID(), taskExecutorIsRegistered, new CompletableFuture<>());
final CompletableFuture<Collection<SlotOffer>> firstOfferResponseFuture = new CompletableFuture<>();
final CompletableFuture<Collection<SlotOffer>> secondOfferResponseFuture = new CompletableFuture<>();
final Queue<CompletableFuture<Collection<SlotOffer>>> slotOfferResponses = new ArrayDeque<>(Arrays.asList(firstOfferResponseFuture, secondOfferResponseFuture));
final MultiShotLatch offerSlotsLatch = new MultiShotLatch();
final TestingJobMasterGateway jobMasterGateway1 = new TestingJobMasterGatewayBuilder().setAddress("jm1").setOfferSlotsFunction((resourceID, slotOffers) -> {
offerSlotsLatch.trigger();
return slotOfferResponses.remove();
}).build();
final TestingJobMasterGateway jobMasterGateway2 = new TestingJobMasterGatewayBuilder().setAddress("jm2").setOfferSlotsFunction((resourceID, slotOffers) -> {
offerSlotsLatch.trigger();
return slotOfferResponses.remove();
}).build();
rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
rpc.registerGateway(jobMasterGateway1.getAddress(), jobMasterGateway1);
rpc.registerGateway(jobMasterGateway2.getAddress(), jobMasterGateway2);
final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
final ThreadSafeTaskSlotTable<Task> threadSafeTaskSlotTable = new ThreadSafeTaskSlotTable<>(taskSlotTable, taskExecutor.getMainThreadExecutableForTesting());
final SlotOffer slotOffer1 = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
final SlotOffer slotOffer2 = new SlotOffer(new AllocationID(), 1, ResourceProfile.ANY);
try {
taskExecutor.start();
taskExecutor.waitUntilStarted();
final TaskExecutorGateway tmGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
// wait until task executor registered at the RM
taskExecutorIsRegistered.await();
// notify job leader to start slot offering
jobManagerLeaderRetriever.notifyListener(jobMasterGateway1.getAddress(), jobMasterGateway1.getFencingToken().toUUID());
jobManagerLeaderRetriever2.notifyListener(jobMasterGateway2.getAddress(), jobMasterGateway2.getFencingToken().toUUID());
// request the first slot
requestSlot(tmGateway, jobId, slotOffer1.getAllocationId(), buildSlotID(slotOffer1.getSlotIndex()), ResourceProfile.UNKNOWN, jobMasterGateway1.getAddress(), resourceManagerGateway.getFencingToken());
// wait until first slot offer as arrived
offerSlotsLatch.await();
// request second slot, triggering another offer containing both slots
requestSlot(tmGateway, jobId2, slotOffer2.getAllocationId(), buildSlotID(slotOffer2.getSlotIndex()), ResourceProfile.UNKNOWN, jobMasterGateway2.getAddress(), resourceManagerGateway.getFencingToken());
// wait until second slot offer as arrived
offerSlotsLatch.await();
firstOfferResponseFuture.complete(Collections.singletonList(slotOffer1));
secondOfferResponseFuture.complete(Collections.singletonList(slotOffer2));
assertThat(threadSafeTaskSlotTable.getActiveTaskSlotAllocationIdsPerJob(jobId), contains(slotOffer1.getAllocationId()));
assertThat(threadSafeTaskSlotTable.getActiveTaskSlotAllocationIdsPerJob(jobId2), contains(slotOffer2.getAllocationId()));
} finally {
RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
}
}
Aggregations