Examples with JobMasterId org.apache.flink.runtime.jobmaster.JobMasterId used on opensource projects

Example 6 with JobMasterId

use of org.apache.flink.runtime.jobmaster.JobMasterId in project flink by apache.

the class DefaultJobLeaderIdServiceTest method testLeaderFutureWaitsForValidLeader.

/**
 * Tests that the leaderId future is only completed once the service is notified about an actual
 * leader being elected. Specifically, it tests that the future is not completed if the
 * leadership was revoked without a new leader having been elected.
 */
@Test(timeout = 10000)
public void testLeaderFutureWaitsForValidLeader() throws Exception {
    final JobID jobId = new JobID();
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null);
    highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);
    JobLeaderIdService jobLeaderIdService = new DefaultJobLeaderIdService(highAvailabilityServices, new ManuallyTriggeredScheduledExecutor(), Time.milliseconds(5000L));
    jobLeaderIdService.start(new NoOpJobLeaderIdActions());
    jobLeaderIdService.addJob(jobId);
    // elect some leader
    leaderRetrievalService.notifyListener("foo", UUID.randomUUID());
    // notify about leadership loss
    leaderRetrievalService.notifyListener(null, null);
    final CompletableFuture<JobMasterId> leaderIdFuture = jobLeaderIdService.getLeaderId(jobId);
    // there is currently no leader, so this should not be completed
    assertThat(leaderIdFuture.isDone(), is(false));
    // elect a new leader
    final UUID newLeaderId = UUID.randomUUID();
    leaderRetrievalService.notifyListener("foo", newLeaderId);
    assertThat(leaderIdFuture.get(), is(JobMasterId.fromUuidOrNull(newLeaderId)));
}

Example 7 with JobMasterId

use of org.apache.flink.runtime.jobmaster.JobMasterId in project flink by apache.

the class DefaultJobLeaderIdServiceTest method testRemovingJob.

/**
 * Tests that removing a job completes the job leader id future exceptionally.
 */
@Test(timeout = 10000)
public void testRemovingJob() throws Exception {
    final JobID jobId = new JobID();
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null);
    highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    Time timeout = Time.milliseconds(5000L);
    JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);
    JobLeaderIdService jobLeaderIdService = new DefaultJobLeaderIdService(highAvailabilityServices, scheduledExecutor, timeout);
    jobLeaderIdService.start(jobLeaderIdActions);
    jobLeaderIdService.addJob(jobId);
    CompletableFuture<JobMasterId> leaderIdFuture = jobLeaderIdService.getLeaderId(jobId);
    // remove the job before we could find a leader
    jobLeaderIdService.removeJob(jobId);
    assertFalse(jobLeaderIdService.containsJob(jobId));
    try {
        leaderIdFuture.get();
        fail("The leader id future should be completed exceptionally.");
    } catch (ExecutionException ignored) {
    // expected exception
    }
}

Example 8 with JobMasterId

use of org.apache.flink.runtime.jobmaster.JobMasterId in project flink by apache.

the class DefaultJobLeaderIdServiceTest method testAddingJob.

/**
 * Tests adding a job and finding out its leader id.
 */
@Test(timeout = 10000)
public void testAddingJob() throws Exception {
    final JobID jobId = new JobID();
    final String address = "foobar";
    final JobMasterId leaderId = JobMasterId.generate();
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null);
    highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    Time timeout = Time.milliseconds(5000L);
    JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);
    JobLeaderIdService jobLeaderIdService = new DefaultJobLeaderIdService(highAvailabilityServices, scheduledExecutor, timeout);
    jobLeaderIdService.start(jobLeaderIdActions);
    jobLeaderIdService.addJob(jobId);
    CompletableFuture<JobMasterId> leaderIdFuture = jobLeaderIdService.getLeaderId(jobId);
    // notify the leader id service about the new leader
    leaderRetrievalService.notifyListener(address, leaderId.toUUID());
    assertEquals(leaderId, leaderIdFuture.get());
    assertTrue(jobLeaderIdService.containsJob(jobId));
}

Example 9 with JobMasterId

use of org.apache.flink.runtime.jobmaster.JobMasterId in project flink by apache.

the class ResourceManagerJobMasterTest method testRegisterJobMasterFromInvalidAddress.

/**
 * Test receive registration with invalid address from job master.
 */
@Test
public void testRegisterJobMasterFromInvalidAddress() throws Exception {
    // test throw exception when receive a registration from job master which takes invalid
    // address
    String invalidAddress = "/jobMasterAddress2";
    CompletableFuture<RegistrationResponse> invalidAddressFuture = resourceManagerGateway.registerJobMaster(new JobMasterId(HighAvailabilityServices.DEFAULT_LEADER_ID), jobMasterResourceId, invalidAddress, jobId, TIMEOUT);
    assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Failure);
}

Example 10 with JobMasterId

use of org.apache.flink.runtime.jobmaster.JobMasterId in project flink by apache.

the class TaskExecutorSubmissionTest method testFailingNotifyPartitionDataAvailable.

/**
 * Test that a failing notifyPartitionDataAvailable call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploying consumer tasks and
 * requiring the invokable to fill one memory segment. The completed memory segment will trigger
 * the scheduling of the downstream operator since it is in pipeline mode. After we've filled
 * the memory segment, we'll block the invokable and wait for the task failure due to the failed
 * notifyPartitionDataAvailable call.
 */
@Test
public void testFailingNotifyPartitionDataAvailable() throws Exception {
    final Configuration configuration = new Configuration();
    // set the memory segment to the smallest size possible, because we have to fill one
    // memory buffer to trigger notifyPartitionDataAvailable to the downstream
    // operators
    configuration.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
    NettyShuffleDescriptor sdd = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
    TaskDeploymentDescriptor tdd = createSender(sdd, TestingAbstractInvokables.TestInvokableRecordCancel.class);
    ExecutionAttemptID eid = tdd.getExecutionAttemptId();
    final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
    final Exception exception = new Exception("Failed notifyPartitionDataAvailable");
    final JobMasterId jobMasterId = JobMasterId.generate();
    TestingJobMasterGateway testingJobMasterGateway = new TestingJobMasterGatewayBuilder().setFencingTokenSupplier(() -> jobMasterId).setNotifyPartitionDataAvailableFunction(resultPartitionID -> FutureUtils.completedExceptionally(exception)).build();
    try (TaskSubmissionTestEnvironment env = new TaskSubmissionTestEnvironment.Builder(jobId).setSlotSize(1).setConfiguration(configuration).addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture).setJobMasterId(jobMasterId).setJobMasterGateway(testingJobMasterGateway).useRealNonMockShuffleEnvironment().build()) {
        TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
        TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();
        TestingAbstractInvokables.TestInvokableRecordCancel.resetGotCanceledFuture();
        taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd, jobMasterId, timeout).get();
        taskRunningFuture.get();
        CompletableFuture<Boolean> cancelFuture = TestingAbstractInvokables.TestInvokableRecordCancel.gotCanceled();
        assertTrue(cancelFuture.get());
        assertTrue(ExceptionUtils.findThrowableWithMessage(taskSlotTable.getTask(eid).getFailureCause(), exception.getMessage()).isPresent());
    }
}