Examples with JobMasterBuilder org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder used on opensource projects

Example 11 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testResourceManagerConnectionAfterStart.

/**
 * Tests that the a JM connects to the leading RM after regaining leadership.
 */
@Test
public void testResourceManagerConnectionAfterStart() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withJobMasterId(jobMasterId).withConfiguration(configuration).withHighAvailabilityServices(haServices).withHeartbeatServices(heartbeatServices).createJobMaster();
    try {
        final TestingResourceManagerGateway testingResourceManagerGateway = createAndRegisterTestingResourceManagerGateway();
        final BlockingQueue<JobMasterId> registrationQueue = new ArrayBlockingQueue<>(1);
        testingResourceManagerGateway.setRegisterJobManagerFunction((jobMasterId, resourceID, s, jobID) -> {
            registrationQueue.offer(jobMasterId);
            return CompletableFuture.completedFuture(testingResourceManagerGateway.getJobMasterRegistrationSuccess());
        });
        notifyResourceManagerLeaderListeners(testingResourceManagerGateway);
        jobMaster.start();
        final JobMasterId firstRegistrationAttempt = registrationQueue.take();
        assertThat(firstRegistrationAttempt, equalTo(jobMasterId));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 12 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterQueryableStateTest method testRequestKvStateWithoutRegistration.

@Test
public void testRequestKvStateWithoutRegistration() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(JOB_GRAPH, rpcService).createJobMaster();
    jobMaster.start();
    final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
    registerSlotsRequiredForJobExecution(jobMasterGateway, JOB_GRAPH.getJobID());
    try {
        // lookup location
        try {
            jobMasterGateway.requestKvStateLocation(JOB_GRAPH.getJobID(), "unknown").get();
            fail("Expected to fail with UnknownKvStateLocation");
        } catch (Exception e) {
            assertTrue(ExceptionUtils.findThrowable(e, UnknownKvStateLocation.class).isPresent());
        }
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 13 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterQueryableStateTest method testRequestKvStateWithIrrelevantRegistration.

@Test
public void testRequestKvStateWithIrrelevantRegistration() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(JOB_GRAPH, rpcService).createJobMaster();
    jobMaster.start();
    final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
    registerSlotsRequiredForJobExecution(jobMasterGateway, JOB_GRAPH.getJobID());
    try {
        // register an irrelevant KvState
        try {
            registerKvState(jobMasterGateway, new JobID(), new JobVertexID(), "any-name");
            fail("Expected to fail with FlinkJobNotFoundException.");
        } catch (Exception e) {
            assertThat(e, containsCause(FlinkJobNotFoundException.class));
        }
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 14 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testTaskManagerRegistrationTriggersHeartbeating.

@Test
public void testTaskManagerRegistrationTriggersHeartbeating() throws Exception {
    final CompletableFuture<ResourceID> heartbeatResourceIdFuture = new CompletableFuture<>();
    final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setHeartbeatJobManagerFunction((taskManagerId, ignored) -> {
        heartbeatResourceIdFuture.complete(taskManagerId);
        return FutureUtils.completedVoidFuture();
    }).createTestingTaskExecutorGateway();
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withResourceId(jmResourceId).withConfiguration(configuration).withHighAvailabilityServices(haServices).withHeartbeatServices(new HeartbeatServices(1L, 10000L)).createJobMaster();
    jobMaster.start();
    try {
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        // register task manager will trigger monitor heartbeat target, schedule heartbeat
        // request at interval time
        CompletableFuture<RegistrationResponse> registrationResponse = jobMasterGateway.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(taskExecutorGateway.getAddress(), unresolvedTaskManagerLocation, TestingUtils.zeroUUID()), testingTimeout);
        // wait for the completion of the registration
        registrationResponse.get();
        assertThat(heartbeatResourceIdFuture.join(), anyOf(nullValue(), equalTo(jmResourceId)));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 15 with JobMasterBuilder

use of org.apache.flink.runtime.jobmaster.utils.JobMasterBuilder in project flink by apache.

the class JobMasterTest method testReleasingTaskExecutorIfNoMoreSlotsRegistered.

/**
 * Tests that the TaskExecutor is released if all of its slots have been freed.
 */
@Test
public void testReleasingTaskExecutorIfNoMoreSlotsRegistered() throws Exception {
    final JobGraph jobGraph = createSingleVertexJobWithRestartStrategy();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withConfiguration(configuration).withHighAvailabilityServices(haServices).withHeartbeatServices(heartbeatServices).createJobMaster();
    final CompletableFuture<JobID> disconnectTaskExecutorFuture = new CompletableFuture<>();
    final CompletableFuture<AllocationID> freedSlotFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction((allocationID, throwable) -> {
        freedSlotFuture.complete(allocationID);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).setDisconnectJobManagerConsumer((jobID, throwable) -> disconnectTaskExecutorFuture.complete(jobID)).createTestingTaskExecutorGateway();
    final LocalUnresolvedTaskManagerLocation taskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    try {
        jobMaster.start();
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        final Collection<SlotOffer> slotOffers = registerSlotsAtJobMaster(1, jobMasterGateway, jobGraph.getJobID(), testingTaskExecutorGateway, taskManagerLocation);
        // check that we accepted the offered slot
        assertThat(slotOffers, hasSize(1));
        final AllocationID allocationId = slotOffers.iterator().next().getAllocationId();
        // now fail the allocation and check that we close the connection to the TaskExecutor
        jobMasterGateway.failSlot(taskManagerLocation.getResourceID(), allocationId, new FlinkException("Fail allocation test exception"));
        // we should free the slot and then disconnect from the TaskExecutor because we use no
        // longer slots from it
        assertThat(freedSlotFuture.get(), equalTo(allocationId));
        assertThat(disconnectTaskExecutorFuture.get(), equalTo(jobGraph.getJobID()));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}