Examples with JobMasterBuilder org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder used on opensource projects

Example 21 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testJobMasterOnlyTerminatesAfterTheSchedulerHasClosed.

@Test
public void testJobMasterOnlyTerminatesAfterTheSchedulerHasClosed() throws Exception {
    final CompletableFuture<Void> schedulerTerminationFuture = new CompletableFuture<>();
    final TestingSchedulerNG testingSchedulerNG = TestingSchedulerNG.newBuilder().setCloseAsyncSupplier(() -> schedulerTerminationFuture).build();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withSlotPoolServiceSchedulerFactory(DefaultSlotPoolServiceSchedulerFactory.create(TestingSlotPoolServiceBuilder.newBuilder(), new TestingSchedulerNGFactory(testingSchedulerNG))).createJobMaster();
    jobMaster.start();
    final CompletableFuture<Void> jobMasterTerminationFuture = jobMaster.closeAsync();
    try {
        jobMasterTerminationFuture.get(10L, TimeUnit.MILLISECONDS);
        fail("Expected TimeoutException because the JobMaster should not terminate.");
    } catch (TimeoutException expected) {
    }
    schedulerTerminationFuture.complete(null);
    jobMasterTerminationFuture.get();
}

Example 22 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testTriggerSavepointTimeout.

/**
 * Tests that the timeout in {@link JobMasterGateway#triggerSavepoint(String, boolean,
 * SavepointFormatType, Time)} is respected.
 */
@Test
public void testTriggerSavepointTimeout() throws Exception {
    final TestingSchedulerNG testingSchedulerNG = TestingSchedulerNG.newBuilder().setTriggerSavepointFunction((ignoredA, ignoredB, formatType) -> new CompletableFuture<>()).build();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withFatalErrorHandler(testingFatalErrorHandler).withSlotPoolServiceSchedulerFactory(DefaultSlotPoolServiceSchedulerFactory.create(TestingSlotPoolServiceBuilder.newBuilder(), new TestingSchedulerNGFactory(testingSchedulerNG))).createJobMaster();
    try {
        jobMaster.start();
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        final CompletableFuture<String> savepointFutureLowTimeout = jobMasterGateway.triggerSavepoint("/tmp", false, SavepointFormatType.CANONICAL, Time.milliseconds(1));
        final CompletableFuture<String> savepointFutureHighTimeout = jobMasterGateway.triggerSavepoint("/tmp", false, SavepointFormatType.CANONICAL, RpcUtils.INF_TIMEOUT);
        try {
            savepointFutureLowTimeout.get(testingTimeout.getSize(), testingTimeout.getUnit());
            fail();
        } catch (final ExecutionException e) {
            final Throwable cause = ExceptionUtils.stripExecutionException(e);
            assertThat(cause, instanceOf(TimeoutException.class));
        }
        assertThat(savepointFutureHighTimeout.isDone(), is(equalTo(false)));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 23 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testJobMasterDisconnectsOldTaskExecutorIfNewSessionIsSeen.

@Test
public void testJobMasterDisconnectsOldTaskExecutorIfNewSessionIsSeen() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    final CompletableFuture<Void> firstTaskExecutorDisconnectedFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway firstTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress("firstTaskExecutor").setDisconnectJobManagerConsumer((jobID, throwable) -> firstTaskExecutorDisconnectedFuture.complete(null)).createTestingTaskExecutorGateway();
    final TestingTaskExecutorGateway secondTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress("secondTaskExecutor").createTestingTaskExecutorGateway();
    rpcService.registerGateway(firstTaskExecutorGateway.getAddress(), firstTaskExecutorGateway);
    rpcService.registerGateway(secondTaskExecutorGateway.getAddress(), secondTaskExecutorGateway);
    try {
        jobMaster.start();
        final LocalUnresolvedTaskManagerLocation taskManagerLocation = new LocalUnresolvedTaskManagerLocation();
        final UUID firstTaskManagerSessionId = UUID.randomUUID();
        final CompletableFuture<RegistrationResponse> firstRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(firstTaskExecutorGateway.getAddress(), taskManagerLocation, firstTaskManagerSessionId), testingTimeout);
        assertThat(firstRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        final UUID secondTaskManagerSessionId = UUID.randomUUID();
        final CompletableFuture<RegistrationResponse> secondRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(secondTaskExecutorGateway.getAddress(), taskManagerLocation, secondTaskManagerSessionId), testingTimeout);
        assertThat(secondRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        // the first TaskExecutor should be disconnected
        firstTaskExecutorDisconnectedFuture.get();
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 24 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method runJobFailureWhenTaskExecutorTerminatesTest.

private void runJobFailureWhenTaskExecutorTerminatesTest(HeartbeatServices heartbeatServices, BiConsumer<LocalUnresolvedTaskManagerLocation, JobMasterGateway> jobReachedRunningState, BiFunction<JobMasterGateway, ResourceID, BiFunction<ResourceID, AllocatedSlotReport, CompletableFuture<Void>>> heartbeatConsumerFunction) throws Exception {
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    final JobMasterBuilder.TestingOnCompletionActions onCompletionActions = new JobMasterBuilder.TestingOnCompletionActions();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withResourceId(jmResourceId).withHighAvailabilityServices(haServices).withHeartbeatServices(heartbeatServices).withOnCompletionActions(onCompletionActions).createJobMaster();
    try {
        jobMaster.start();
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        final LocalUnresolvedTaskManagerLocation taskManagerUnresolvedLocation = new LocalUnresolvedTaskManagerLocation();
        final CompletableFuture<ExecutionAttemptID> taskDeploymentFuture = new CompletableFuture<>();
        final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
            taskDeploymentFuture.complete(taskDeploymentDescriptor.getExecutionAttemptId());
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).setHeartbeatJobManagerFunction(heartbeatConsumerFunction.apply(jobMasterGateway, taskManagerUnresolvedLocation.getResourceID())).createTestingTaskExecutorGateway();
        final Collection<SlotOffer> slotOffers = registerSlotsAtJobMaster(1, jobMasterGateway, jobGraph.getJobID(), taskExecutorGateway, taskManagerUnresolvedLocation);
        assertThat(slotOffers, hasSize(1));
        final ExecutionAttemptID executionAttemptId = taskDeploymentFuture.get();
        jobMasterGateway.updateTaskExecutionState(new TaskExecutionState(executionAttemptId, ExecutionState.INITIALIZING)).get();
        jobMasterGateway.updateTaskExecutionState(new TaskExecutionState(executionAttemptId, ExecutionState.RUNNING)).get();
        jobReachedRunningState.accept(taskManagerUnresolvedLocation, jobMasterGateway);
        final ArchivedExecutionGraph archivedExecutionGraph = onCompletionActions.getJobReachedGloballyTerminalStateFuture().get().getArchivedExecutionGraph();
        assertThat(archivedExecutionGraph.getState(), is(JobStatus.FAILED));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 25 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testJobMasterAcknowledgesDuplicateTaskExecutorRegistrations.

@Test
public void testJobMasterAcknowledgesDuplicateTaskExecutorRegistrations() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    rpcService.registerGateway(testingTaskExecutorGateway.getAddress(), testingTaskExecutorGateway);
    try {
        jobMaster.start();
        final TaskManagerRegistrationInformation taskManagerRegistrationInformation = TaskManagerRegistrationInformation.create(testingTaskExecutorGateway.getAddress(), new LocalUnresolvedTaskManagerLocation(), UUID.randomUUID());
        final CompletableFuture<RegistrationResponse> firstRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), taskManagerRegistrationInformation, testingTimeout);
        final CompletableFuture<RegistrationResponse> secondRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), taskManagerRegistrationInformation, testingTimeout);
        assertThat(firstRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        assertThat(secondRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}