Examples with TaskExecutorGateway org.apache.flink.runtime.taskexecutor.TaskExecutorGateway used on opensource projects

Example 16 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class RpcConnectionTest method testConnectFailure.

@Test
public void testConnectFailure() throws Exception {
    // we start the RPC service with a very long timeout to ensure that the test
    // can only pass if the connection problem is not recognized merely via a timeout
    Configuration configuration = new Configuration();
    configuration.set(AkkaOptions.ASK_TIMEOUT_DURATION, Duration.ofSeconds(10000000));
    final RpcService rpcService = RpcSystem.load().localServiceBuilder(configuration).withBindAddress("localhost").withBindPort(0).createAndStart();
    try {
        CompletableFuture<TaskExecutorGateway> future = rpcService.connect("foo.bar.com.test.invalid", TaskExecutorGateway.class);
        future.get(10000000, TimeUnit.SECONDS);
        fail("should never complete normally");
    } catch (TimeoutException e) {
        fail("should not fail with a generic timeout exception");
    } catch (ExecutionException e) {
        // that is what we want
        assertTrue(e.getCause() instanceof RpcConnectionException);
        assertTrue("wrong error message", e.getCause().getMessage().contains("foo.bar.com.test.invalid"));
    } catch (Throwable t) {
        fail("wrong exception: " + t);
    } finally {
        rpcService.stopService().get();
    }
}

Example 17 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class FineGrainedSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all allocated slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final AllocationID allocationId = new AllocationID();
    final SlotReport slotReport = new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE));
    new Context() {

        {
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture = new CompletableFuture<>();
                final CompletableFuture<Boolean> unRegisterTaskManagerFuture = new CompletableFuture<>();
                runInMainThread(() -> registerTaskManagerFuture.complete(getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers().size(), is(1));
                final Optional<TaskManagerSlotInformation> slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId);
                assertTrue(slot.isPresent());
                assertTrue(slot.get().getState() == SlotState.ALLOCATED);
                runInMainThread(() -> unRegisterTaskManagerFuture.complete(getSlotManager().unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION)));
                assertThat(assertFutureCompleteAndReturn(unRegisterTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers(), is(empty()));
                assertFalse(getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId).isPresent());
            });
        }
    };
}

Example 18 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class JobMasterTest method registerSlotsAtJobMaster.

private Collection<SlotOffer> registerSlotsAtJobMaster(int numberSlots, JobMasterGateway jobMasterGateway, JobID jobId, TaskExecutorGateway taskExecutorGateway, UnresolvedTaskManagerLocation unresolvedTaskManagerLocation) throws ExecutionException, InterruptedException {
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    jobMasterGateway.registerTaskManager(jobId, TaskManagerRegistrationInformation.create(taskExecutorGateway.getAddress(), unresolvedTaskManagerLocation, TestingUtils.zeroUUID()), testingTimeout).get();
    Collection<SlotOffer> slotOffers = IntStream.range(0, numberSlots).mapToObj(index -> new SlotOffer(new AllocationID(), index, ResourceProfile.ANY)).collect(Collectors.toList());
    return jobMasterGateway.offerSlots(unresolvedTaskManagerLocation.getResourceID(), slotOffers, testingTimeout).get();
}

Example 19 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class JobMasterExecutionDeploymentReconciliationTest method testExecutionDeploymentReconciliation.

/**
 * Tests how the job master handles unknown/missing executions.
 */
@Test
public void testExecutionDeploymentReconciliation() throws Exception {
    JobMasterBuilder.TestingOnCompletionActions onCompletionActions = new JobMasterBuilder.TestingOnCompletionActions();
    TestingExecutionDeploymentTrackerWrapper deploymentTrackerWrapper = new TestingExecutionDeploymentTrackerWrapper();
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    JobMaster jobMaster = createAndStartJobMaster(onCompletionActions, deploymentTrackerWrapper, jobGraph);
    JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
    RPC_SERVICE_RESOURCE.getTestingRpcService().registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final CompletableFuture<ExecutionAttemptID> taskCancellationFuture = new CompletableFuture<>();
    TaskExecutorGateway taskExecutorGateway = createTaskExecutorGateway(taskCancellationFuture);
    LocalUnresolvedTaskManagerLocation localUnresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    registerTaskExecutorAndOfferSlots(jobMasterGateway, jobGraph.getJobID(), taskExecutorGateway, localUnresolvedTaskManagerLocation);
    ExecutionAttemptID deployedExecution = deploymentTrackerWrapper.getTaskDeploymentFuture().get();
    assertFalse(taskCancellationFuture.isDone());
    ExecutionAttemptID unknownDeployment = new ExecutionAttemptID();
    // the deployment report is missing the just deployed task, but contains the ID of some
    // other unknown deployment
    // the job master should cancel the unknown deployment, and fail the job
    jobMasterGateway.heartbeatFromTaskManager(localUnresolvedTaskManagerLocation.getResourceID(), new TaskExecutorToJobManagerHeartbeatPayload(new AccumulatorReport(Collections.emptyList()), new ExecutionDeploymentReport(Collections.singleton(unknownDeployment))));
    assertThat(taskCancellationFuture.get(), is(unknownDeployment));
    assertThat(deploymentTrackerWrapper.getStopFuture().get(), is(deployedExecution));
    assertThat(onCompletionActions.getJobReachedGloballyTerminalStateFuture().get().getArchivedExecutionGraph().getState(), is(JobStatus.FAILED));
}

Example 20 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class JobMasterExecutionDeploymentReconciliationTest method createTaskExecutorGateway.

private TaskExecutorGateway createTaskExecutorGateway(CompletableFuture<ExecutionAttemptID> taskCancellationFuture, CompletableFuture<ExecutionAttemptID> taskSubmissionFuture, CompletableFuture<Acknowledge> taskSubmissionResponse) {
    TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).setCancelTaskFunction(executionAttemptId -> {
        taskCancellationFuture.complete(executionAttemptId);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).setSubmitTaskConsumer((tdd, ignored) -> {
        taskSubmissionFuture.complete(tdd.getExecutionAttemptId());
        return taskSubmissionResponse;
    }).createTestingTaskExecutorGateway();
    RPC_SERVICE_RESOURCE.getTestingRpcService().registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    return taskExecutorGateway;
}