Examples with TestingResourceManagerGateway org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway used on opensource projects

Example 26 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class ExecutionGraphRestartTest method setupSlotPool.

private static void setupSlotPool(SlotPool slotPool) throws Exception {
    final String jobManagerAddress = "foobar";
    final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
    slotPool.connectToResourceManager(resourceManagerGateway);
}

Example 27 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testSlotAcceptance.

/**
 * Tests that accepted slots go into state assigned and the others are returned to the resource
 * manager.
 */
@Test
public void testSlotAcceptance() throws Exception {
    final InstanceID registrationId = new InstanceID();
    final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
    final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);
    final OneShotLatch offerSlotsLatch = new OneShotLatch();
    final OneShotLatch taskInTerminalState = new OneShotLatch();
    final CompletableFuture<Collection<SlotOffer>> offerResultFuture = new CompletableFuture<>();
    final TestingJobMasterGateway jobMasterGateway = createJobMasterWithSlotOfferAndTaskTerminationHooks(offerSlotsLatch, taskInTerminalState, offerResultFuture);
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
    final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
    final TestingTaskExecutor taskManager = createTestingTaskExecutor(taskManagerServices);
    try {
        taskManager.start();
        taskManager.waitUntilStarted();
        final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);
        // wait until registered at the RM
        taskExecutorIsRegistered.await();
        // request 2 slots for the given allocation ids
        AllocationID[] allocationIds = new AllocationID[] { allocationId1, allocationId2 };
        for (int i = 0; i < allocationIds.length; i++) {
            requestSlot(tmGateway, jobId, allocationIds[i], buildSlotID(i), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        }
        // notify job leader to start slot offering
        jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        // wait until slots have been offered
        offerSlotsLatch.await();
        offerResultFuture.complete(Collections.singletonList(offer1));
        final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();
        final Tuple3<InstanceID, SlotID, AllocationID> expectedResult = Tuple3.of(registrationId, buildSlotID(1), allocationId2);
        assertThat(instanceIDSlotIDAllocationIDTuple3, equalTo(expectedResult));
        // the slot 1 can be activate for task submission
        submit(allocationId1, jobMasterGateway, tmGateway, NoOpInvokable.class);
        // wait for the task completion
        taskInTerminalState.await();
        // the slot 2 can NOT be activate for task submission
        try {
            submit(allocationId2, jobMasterGateway, tmGateway, NoOpInvokable.class);
            fail("It should not be possible to submit task to acquired by JM slot with index 1 (allocationId2)");
        } catch (CompletionException e) {
            assertThat(e.getCause(), instanceOf(TaskSubmissionException.class));
        }
        // the slot 2 is free to request
        requestSlot(tmGateway, jobId, allocationId2, buildSlotID(1), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
    } finally {
        RpcUtils.terminateRpcEndpoint(taskManager, timeout);
    }
}

Example 28 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testReleaseInactiveSlots.

@Test
public void testReleaseInactiveSlots() throws Exception {
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1)).build();
    final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices, HEARTBEAT_SERVICES);
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setRegisterTaskManagerFunction((ignoredJobId, ignoredTaskManagerRegistrationInformation) -> new CompletableFuture<>()).build();
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final InstanceID registrationId = new InstanceID();
    final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
    final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    resourceManagerLeaderRetriever.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
    try {
        taskExecutor.start();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        taskExecutorIsRegistered.await();
        final AllocationID allocationId = new AllocationID();
        final SlotID slotId = new SlotID(taskExecutor.getResourceID(), 0);
        requestSlot(taskExecutorGateway, jobId, allocationId, slotId, ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        taskExecutorGateway.freeInactiveSlots(jobId, timeout);
        // the slot should be freed
        assertThat(availableSlotFuture.get().f1, is(slotId));
        assertThat(availableSlotFuture.get().f2, is(allocationId));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}

Example 29 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testImmediatelyRegistersIfLeaderIsKnown.

@Test
public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
    final String resourceManagerAddress = "/resource/manager/address/one";
    final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
    final CountDownLatch taskManagerRegisteredLatch = new CountDownLatch(1);
    testingResourceManagerGateway.setRegisterTaskExecutorFunction(FunctionUtils.uncheckedFunction(ignored -> {
        taskManagerRegisteredLatch.countDown();
        return CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(new InstanceID(), new ResourceID(resourceManagerAddress), new ClusterInformation("localhost", 1234)));
    }));
    rpc.registerGateway(resourceManagerAddress, testingResourceManagerGateway);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation).setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1)).setTaskStateManager(createTaskExecutorLocalStateStoresManager()).build();
    final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);
    try {
        taskManager.start();
        resourceManagerLeaderRetriever.notifyListener(resourceManagerAddress, UUID.randomUUID());
        assertTrue(taskManagerRegisteredLatch.await(timeout.toMilliseconds(), TimeUnit.MILLISECONDS));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskManager, timeout);
    }
}

Example 30 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testSlotOfferResponseWithPendingSlotOffer.

/**
 * Tests the behavior of the task executor when a slot offer response is received while a newer
 * slot offer is in progress.
 */
private void testSlotOfferResponseWithPendingSlotOffer(final ResponseOrder responseOrder) throws Exception {
    final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
    final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(new InstanceID(), taskExecutorIsRegistered, new CompletableFuture<>());
    final CompletableFuture<Collection<SlotOffer>> firstOfferResponseFuture = new CompletableFuture<>();
    final CompletableFuture<Collection<SlotOffer>> secondOfferResponseFuture = new CompletableFuture<>();
    final Queue<CompletableFuture<Collection<SlotOffer>>> slotOfferResponses = new ArrayDeque<>(Arrays.asList(firstOfferResponseFuture, secondOfferResponseFuture));
    final MultiShotLatch offerSlotsLatch = new MultiShotLatch();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        offerSlotsLatch.trigger();
        return slotOfferResponses.remove();
    }).build();
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
    final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
    final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
    final ThreadSafeTaskSlotTable<Task> threadSafeTaskSlotTable = new ThreadSafeTaskSlotTable<>(taskSlotTable, taskExecutor.getMainThreadExecutableForTesting());
    final SlotOffer slotOffer1 = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
    final SlotOffer slotOffer2 = new SlotOffer(new AllocationID(), 1, ResourceProfile.ANY);
    try {
        taskExecutor.start();
        taskExecutor.waitUntilStarted();
        final TaskExecutorGateway tmGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        // wait until task executor registered at the RM
        taskExecutorIsRegistered.await();
        // notify job leader to start slot offering
        jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        // request the first slot
        requestSlot(tmGateway, jobId, slotOffer1.getAllocationId(), buildSlotID(slotOffer1.getSlotIndex()), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        // wait until first slot offer as arrived
        offerSlotsLatch.await();
        // request second slot, triggering another offer containing both slots
        int slotIndex = slotOffer2.getSlotIndex();
        requestSlot(tmGateway, jobId, slotOffer2.getAllocationId(), buildSlotID(slotIndex), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        // wait until second slot offer as arrived
        offerSlotsLatch.await();
        switch(responseOrder) {
            case ACCEPT_THEN_REJECT:
                // accept the first offer, but reject both slots for the second offer
                firstOfferResponseFuture.complete(Collections.singletonList(slotOffer1));
                assertThat(threadSafeTaskSlotTable.getActiveTaskSlotAllocationIdsPerJob(jobId), empty());
                secondOfferResponseFuture.complete(Collections.emptyList());
                assertThat(threadSafeTaskSlotTable.getAllocationIdsPerJob(jobId), empty());
                return;
            case REJECT_THEN_ACCEPT:
                // fail the first offer, but accept both slots for the second offer
                // in the past the rejection of the first offer freed the slot; when the slot is
                // accepted from the second offer the activation of said slot then failed
                firstOfferResponseFuture.complete(Collections.emptyList());
                secondOfferResponseFuture.complete(Arrays.asList(slotOffer1, slotOffer2));
                assertThat(threadSafeTaskSlotTable.getAllocationIdsPerJob(jobId), containsInAnyOrder(slotOffer1.getAllocationId(), slotOffer2.getAllocationId()));
                return;
        }
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}