Examples with TestingResourceManagerGateway org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway used on opensource projects

Example 16 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method createRmWithTmRegisterAndNotifySlotHooks.

private TestingResourceManagerGateway createRmWithTmRegisterAndNotifySlotHooks(InstanceID registrationId, OneShotLatch taskExecutorIsRegistered, CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture) {
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    resourceManagerLeaderRetriever.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
    resourceManagerGateway.setRegisterTaskExecutorFunction(taskExecutorRegistration -> CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(registrationId, resourceManagerGateway.getOwnResourceId(), new ClusterInformation("localhost", 1234))));
    resourceManagerGateway.setNotifySlotAvailableConsumer(availableSlotFuture::complete);
    resourceManagerGateway.setSendSlotReportFunction(ignored -> {
        taskExecutorIsRegistered.trigger();
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    return resourceManagerGateway;
}

Example 17 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testSubmitTaskBeforeAcceptSlot.

/**
 * This tests task executor receive SubmitTask before OfferSlot response.
 */
@Test
public void testSubmitTaskBeforeAcceptSlot() throws Exception {
    final InstanceID registrationId = new InstanceID();
    final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
    final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway resourceManagerGateway = createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);
    final OneShotLatch offerSlotsLatch = new OneShotLatch();
    final OneShotLatch taskInTerminalState = new OneShotLatch();
    final CompletableFuture<Collection<SlotOffer>> offerResultFuture = new CompletableFuture<>();
    final TestingJobMasterGateway jobMasterGateway = createJobMasterWithSlotOfferAndTaskTerminationHooks(offerSlotsLatch, taskInTerminalState, offerResultFuture);
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
    final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
    final TestingTaskExecutor taskManager = createTestingTaskExecutor(taskManagerServices);
    try {
        taskManager.start();
        taskManager.waitUntilStarted();
        final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);
        // wait until registered at the RM
        taskExecutorIsRegistered.await();
        // request 2 slots for the given allocation ids
        AllocationID[] allocationIds = new AllocationID[] { allocationId1, allocationId2 };
        for (int i = 0; i < allocationIds.length; i++) {
            requestSlot(tmGateway, jobId, allocationIds[i], buildSlotID(i), ResourceProfile.UNKNOWN, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        }
        // notify job leader to start slot offering
        jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        // wait until slots have been offered
        offerSlotsLatch.await();
        submit(allocationId1, jobMasterGateway, tmGateway, NoOpInvokable.class);
        // acknowledge the offered slots
        offerResultFuture.complete(Collections.singleton(offer1));
        // check that the rejected slot will be made available again
        final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();
        assertThat(instanceIDSlotIDAllocationIDTuple3.f2, equalTo(allocationId2));
        // wait for the task completion
        taskInTerminalState.await();
    } finally {
        RpcUtils.terminateRpcEndpoint(taskManager, timeout);
    }
}

Example 18 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testJobLeaderDetection.

/**
 * Tests that a TaskManager detects a job leader for which it has reserved slots. Upon detecting
 * the job leader, it will offer all reserved slots to the JobManager.
 */
@Test
public void testJobLeaderDetection() throws Exception {
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);
    final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    CompletableFuture<Void> initialSlotReportFuture = new CompletableFuture<>();
    resourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        initialSlotReportFuture.complete(null);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final CompletableFuture<Collection<SlotOffer>> offeredSlotsFuture = new CompletableFuture<>();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        offeredSlotsFuture.complete(new ArrayList<>(slotOffers));
        return CompletableFuture.completedFuture(slotOffers);
    }).build();
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final AllocationID allocationId = new AllocationID();
    final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation).setTaskSlotTable(taskSlotTable).setJobLeaderService(jobLeaderService).setTaskStateManager(localStateStoresManager).build();
    TaskExecutor taskManager = createTaskExecutor(taskManagerServices);
    try {
        taskManager.start();
        final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);
        // tell the task manager about the rm leader
        resourceManagerLeaderRetriever.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
        initialSlotReportFuture.get();
        requestSlot(tmGateway, jobId, allocationId, buildSlotID(0), ResourceProfile.ZERO, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        // now inform the task manager about the new job leader
        jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        final Collection<SlotOffer> offeredSlots = offeredSlotsFuture.get();
        final Collection<AllocationID> allocationIds = offeredSlots.stream().map(SlotOffer::getAllocationId).collect(Collectors.toList());
        assertThat(allocationIds, containsInAnyOrder(allocationId));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskManager, timeout);
    }
}

Example 19 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testMaximumRegistrationDurationAfterConnectionLoss.

@Test
public void testMaximumRegistrationDurationAfterConnectionLoss() throws Exception {
    configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("100 ms"));
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();
    final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices, new HeartbeatServices(10L, 10L));
    taskExecutor.start();
    final CompletableFuture<ResourceID> registrationFuture = new CompletableFuture<>();
    final OneShotLatch secondRegistration = new OneShotLatch();
    try {
        final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
        testingResourceManagerGateway.setRegisterTaskExecutorFunction(taskExecutorRegistration -> {
            if (registrationFuture.complete(taskExecutorRegistration.getResourceId())) {
                return createRegistrationResponse(testingResourceManagerGateway);
            } else {
                secondRegistration.trigger();
                return CompletableFuture.completedFuture(new Failure(new FlinkException("Only the first registration should succeed.")));
            }
        });
        rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
        resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), UUID.randomUUID());
        final ResourceID registrationResourceId = registrationFuture.get();
        assertThat(registrationResourceId, equalTo(taskManagerServices.getUnresolvedTaskManagerLocation().getResourceID()));
        secondRegistration.await();
        final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
        assertThat(error, is(notNullValue()));
        assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));
        testingFatalErrorHandler.clearError();
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}

Example 20 with TestingResourceManagerGateway

use of org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway in project flink by apache.

the class TaskExecutorTest method testInitialSlotReport.

/**
 * Tests that the {@link TaskExecutor} sends the initial slot report after it registered at the
 * ResourceManager.
 */
@Test
public void testInitialSlotReport() throws Exception {
    final TaskExecutor taskExecutor = createTaskExecutor(1);
    taskExecutor.start();
    try {
        final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
        final CompletableFuture<ResourceID> initialSlotReportFuture = new CompletableFuture<>();
        testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
            initialSlotReportFuture.complete(resourceIDInstanceIDSlotReportTuple3.f0);
            return CompletableFuture.completedFuture(Acknowledge.get());
        });
        rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
        resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());
        assertThat(initialSlotReportFuture.get(), equalTo(taskExecutor.getResourceID()));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}