Examples with TaskExecutorGateway org.apache.flink.runtime.taskexecutor.TaskExecutorGateway used on opensource projects

Example 1 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class JobMaster method offerSlots.

@RpcMethod
public Future<Iterable<SlotOffer>> offerSlots(final ResourceID taskManagerId, final Iterable<SlotOffer> slots, final UUID leaderId) throws Exception {
    validateLeaderSessionId(leaderId);
    Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);
    if (taskManager == null) {
        throw new Exception("Unknown TaskManager " + taskManagerId);
    }
    final JobID jid = jobGraph.getJobID();
    final TaskManagerLocation taskManagerLocation = taskManager.f0;
    final TaskExecutorGateway taskExecutorGateway = taskManager.f1;
    final ArrayList<Tuple2<AllocatedSlot, SlotOffer>> slotsAndOffers = new ArrayList<>();
    final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, leaderId);
    for (SlotOffer slotOffer : slots) {
        final AllocatedSlot slot = new AllocatedSlot(slotOffer.getAllocationId(), jid, taskManagerLocation, slotOffer.getSlotIndex(), slotOffer.getResourceProfile(), rpcTaskManagerGateway);
        slotsAndOffers.add(new Tuple2<>(slot, slotOffer));
    }
    return slotPoolGateway.offerSlots(slotsAndOffers);
}

Example 2 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class SlotProtocolTest method testSlotAvailableRequest.

/**
	 * Tests whether
	 * 1) a SlotRequest is routed to the SlotManager
	 * 2) a SlotRequest is confirmed
	 * 3) a SlotRequest leads to an allocation of a registered slot
	 * 4) a SlotRequest is routed to the TaskExecutor
	 */
@Test
public void testSlotAvailableRequest() throws Exception {
    final String rmAddress = "/rm1";
    final String jmAddress = "/jm1";
    final String tmAddress = "/tm1";
    final JobID jobID = new JobID();
    testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
    final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
    final UUID rmLeaderID = UUID.randomUUID();
    final UUID jmLeaderID = UUID.randomUUID();
    TestingLeaderElectionService rmLeaderElectionService = configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
    TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
    Mockito.when(taskExecutorGateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
    testRpcService.registerGateway(tmAddress, taskExecutorGateway);
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices, testRpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
    ResourceManager<ResourceID> resourceManager = Mockito.spy(new StandaloneResourceManager(testRpcService, resourceManagerConfiguration, testingHaServices, slotManagerFactory, mock(MetricRegistry.class), jobLeaderIdService, mock(FatalErrorHandler.class)));
    resourceManager.start();
    rmLeaderElectionService.isLeader(rmLeaderID);
    Thread.sleep(1000);
    Future<RegistrationResponse> registrationFuture = resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
    try {
        registrationFuture.get(5L, TimeUnit.SECONDS);
    } catch (Exception e) {
        Assert.fail("JobManager registration Future didn't become ready.");
    }
    final SlotManager slotManager = slotManagerFactory.slotManager;
    final ResourceID resourceID = ResourceID.generate();
    final AllocationID allocationID = new AllocationID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
    final SlotID slotID = new SlotID(resourceID, 0);
    final SlotStatus slotStatus = new SlotStatus(slotID, resourceProfile);
    final SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    // register slot at SlotManager
    slotManager.registerTaskExecutor(resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
    SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
    RMSlotRequestReply slotRequestReply = resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
    // 1) a SlotRequest is routed to the SlotManager
    verify(slotManager).requestSlot(slotRequest);
    // 2) a SlotRequest is confirmed
    Assert.assertEquals(slotRequestReply.getAllocationID(), allocationID);
    // 3) a SlotRequest leads to an allocation of a registered slot
    Assert.assertTrue(slotManager.isAllocated(slotID));
    Assert.assertTrue(slotManager.isAllocated(allocationID));
    // 4) a SlotRequest is routed to the TaskExecutor
    verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}

Example 3 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class SlotManagerTest method setUp.

@BeforeClass
public static void setUp() {
    taskExecutorRegistration = Mockito.mock(TaskExecutorRegistration.class);
    TaskExecutorGateway gateway = Mockito.mock(TaskExecutorGateway.class);
    Mockito.when(taskExecutorRegistration.getTaskExecutorGateway()).thenReturn(gateway);
    Mockito.when(gateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
}

Example 4 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class ResourceManager method registerTaskExecutor.

@Override
public CompletableFuture<RegistrationResponse> registerTaskExecutor(final TaskExecutorRegistration taskExecutorRegistration, final Time timeout) {
    CompletableFuture<TaskExecutorGateway> taskExecutorGatewayFuture = getRpcService().connect(taskExecutorRegistration.getTaskExecutorAddress(), TaskExecutorGateway.class);
    taskExecutorGatewayFutures.put(taskExecutorRegistration.getResourceId(), taskExecutorGatewayFuture);
    return taskExecutorGatewayFuture.handleAsync((TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
        final ResourceID resourceId = taskExecutorRegistration.getResourceId();
        if (taskExecutorGatewayFuture == taskExecutorGatewayFutures.get(resourceId)) {
            taskExecutorGatewayFutures.remove(resourceId);
            if (throwable != null) {
                return new RegistrationResponse.Failure(throwable);
            } else {
                return registerTaskExecutorInternal(taskExecutorGateway, taskExecutorRegistration);
            }
        } else {
            log.debug("Ignoring outdated TaskExecutorGateway connection for {}.", resourceId.getStringWithMetadata());
            return new RegistrationResponse.Failure(new FlinkException("Decline outdated task executor registration."));
        }
    }, getMainThreadExecutor());
}

Example 5 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManager method maybeReclaimInactiveSlots.

private void maybeReclaimInactiveSlots(JobID jobId) {
    if (!resourceTracker.getAcquiredResources(jobId).isEmpty()) {
        final Collection<TaskExecutorConnection> taskExecutorsWithAllocatedSlots = slotTracker.getTaskExecutorsWithAllocatedSlotsForJob(jobId);
        for (TaskExecutorConnection taskExecutorConnection : taskExecutorsWithAllocatedSlots) {
            final TaskExecutorGateway taskExecutorGateway = taskExecutorConnection.getTaskExecutorGateway();
            taskExecutorGateway.freeInactiveSlots(jobId, taskManagerRequestTimeout);
        }
    }
}