use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.
the class JobMaster method offerSlots.
@RpcMethod
public Future<Iterable<SlotOffer>> offerSlots(final ResourceID taskManagerId, final Iterable<SlotOffer> slots, final UUID leaderId) throws Exception {
validateLeaderSessionId(leaderId);
Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);
if (taskManager == null) {
throw new Exception("Unknown TaskManager " + taskManagerId);
}
final JobID jid = jobGraph.getJobID();
final TaskManagerLocation taskManagerLocation = taskManager.f0;
final TaskExecutorGateway taskExecutorGateway = taskManager.f1;
final ArrayList<Tuple2<AllocatedSlot, SlotOffer>> slotsAndOffers = new ArrayList<>();
final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, leaderId);
for (SlotOffer slotOffer : slots) {
final AllocatedSlot slot = new AllocatedSlot(slotOffer.getAllocationId(), jid, taskManagerLocation, slotOffer.getSlotIndex(), slotOffer.getResourceProfile(), rpcTaskManagerGateway);
slotsAndOffers.add(new Tuple2<>(slot, slotOffer));
}
return slotPoolGateway.offerSlots(slotsAndOffers);
}
use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.
the class SlotProtocolTest method testSlotAvailableRequest.
/**
* Tests whether
* 1) a SlotRequest is routed to the SlotManager
* 2) a SlotRequest is confirmed
* 3) a SlotRequest leads to an allocation of a registered slot
* 4) a SlotRequest is routed to the TaskExecutor
*/
@Test
public void testSlotAvailableRequest() throws Exception {
final String rmAddress = "/rm1";
final String jmAddress = "/jm1";
final String tmAddress = "/tm1";
final JobID jobID = new JobID();
testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
final UUID rmLeaderID = UUID.randomUUID();
final UUID jmLeaderID = UUID.randomUUID();
TestingLeaderElectionService rmLeaderElectionService = configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
Mockito.when(taskExecutorGateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices, testRpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
ResourceManager<ResourceID> resourceManager = Mockito.spy(new StandaloneResourceManager(testRpcService, resourceManagerConfiguration, testingHaServices, slotManagerFactory, mock(MetricRegistry.class), jobLeaderIdService, mock(FatalErrorHandler.class)));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
Thread.sleep(1000);
Future<RegistrationResponse> registrationFuture = resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
try {
registrationFuture.get(5L, TimeUnit.SECONDS);
} catch (Exception e) {
Assert.fail("JobManager registration Future didn't become ready.");
}
final SlotManager slotManager = slotManagerFactory.slotManager;
final ResourceID resourceID = ResourceID.generate();
final AllocationID allocationID = new AllocationID();
final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
final SlotID slotID = new SlotID(resourceID, 0);
final SlotStatus slotStatus = new SlotStatus(slotID, resourceProfile);
final SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
// register slot at SlotManager
slotManager.registerTaskExecutor(resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
RMSlotRequestReply slotRequestReply = resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
// 1) a SlotRequest is routed to the SlotManager
verify(slotManager).requestSlot(slotRequest);
// 2) a SlotRequest is confirmed
Assert.assertEquals(slotRequestReply.getAllocationID(), allocationID);
// 3) a SlotRequest leads to an allocation of a registered slot
Assert.assertTrue(slotManager.isAllocated(slotID));
Assert.assertTrue(slotManager.isAllocated(allocationID));
// 4) a SlotRequest is routed to the TaskExecutor
verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}
use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.
the class SlotManagerTest method setUp.
@BeforeClass
public static void setUp() {
taskExecutorRegistration = Mockito.mock(TaskExecutorRegistration.class);
TaskExecutorGateway gateway = Mockito.mock(TaskExecutorGateway.class);
Mockito.when(taskExecutorRegistration.getTaskExecutorGateway()).thenReturn(gateway);
Mockito.when(gateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
}
use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.
the class ResourceManager method registerTaskExecutor.
@Override
public CompletableFuture<RegistrationResponse> registerTaskExecutor(final TaskExecutorRegistration taskExecutorRegistration, final Time timeout) {
CompletableFuture<TaskExecutorGateway> taskExecutorGatewayFuture = getRpcService().connect(taskExecutorRegistration.getTaskExecutorAddress(), TaskExecutorGateway.class);
taskExecutorGatewayFutures.put(taskExecutorRegistration.getResourceId(), taskExecutorGatewayFuture);
return taskExecutorGatewayFuture.handleAsync((TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
final ResourceID resourceId = taskExecutorRegistration.getResourceId();
if (taskExecutorGatewayFuture == taskExecutorGatewayFutures.get(resourceId)) {
taskExecutorGatewayFutures.remove(resourceId);
if (throwable != null) {
return new RegistrationResponse.Failure(throwable);
} else {
return registerTaskExecutorInternal(taskExecutorGateway, taskExecutorRegistration);
}
} else {
log.debug("Ignoring outdated TaskExecutorGateway connection for {}.", resourceId.getStringWithMetadata());
return new RegistrationResponse.Failure(new FlinkException("Decline outdated task executor registration."));
}
}, getMainThreadExecutor());
}
use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.
the class DeclarativeSlotManager method maybeReclaimInactiveSlots.
private void maybeReclaimInactiveSlots(JobID jobId) {
if (!resourceTracker.getAcquiredResources(jobId).isEmpty()) {
final Collection<TaskExecutorConnection> taskExecutorsWithAllocatedSlots = slotTracker.getTaskExecutorsWithAllocatedSlotsForJob(jobId);
for (TaskExecutorConnection taskExecutorConnection : taskExecutorsWithAllocatedSlots) {
final TaskExecutorGateway taskExecutorGateway = taskExecutorConnection.getTaskExecutorGateway();
taskExecutorGateway.freeInactiveSlots(jobId, taskManagerRequestTimeout);
}
}
}
Aggregations