Examples with JobResult org.apache.flink.runtime.jobmaster.JobResult used on opensource projects

Example 31 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class DispatcherTest method testJobSuspensionWhenDispatcherIsTerminated.

/**
 * Tests that a submitted job is suspended if the Dispatcher is terminated.
 */
@Test
public void testJobSuspensionWhenDispatcherIsTerminated() throws Exception {
    dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(jobId, createdJobManagerRunnerLatch));
    DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);
    dispatcherGateway.submitJob(jobGraph, TIMEOUT).get();
    final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway.requestJobResult(jobGraph.getJobID(), TIMEOUT);
    assertThat(jobResultFuture.isDone(), is(false));
    dispatcher.close();
    final JobResult jobResult = jobResultFuture.get();
    assertEquals(jobResult.getApplicationStatus(), ApplicationStatus.UNKNOWN);
}

Example 32 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class DispatcherTest method testDuplicateJobSubmissionWithGloballyTerminatedJobId.

@Test
public void testDuplicateJobSubmissionWithGloballyTerminatedJobId() throws Exception {
    final JobResult jobResult = TestingJobResultStore.createJobResult(jobGraph.getJobID(), ApplicationStatus.SUCCEEDED);
    haServices.getJobResultStore().createDirtyResult(new JobResultEntry(jobResult));
    dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(jobId, createdJobManagerRunnerLatch));
    final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);
    final CompletableFuture<Acknowledge> submitFuture = dispatcherGateway.submitJob(jobGraph, TIMEOUT);
    final ExecutionException executionException = assertThrows(ExecutionException.class, submitFuture::get);
    assertTrue(executionException.getCause() instanceof DuplicateJobSubmissionException);
    final DuplicateJobSubmissionException duplicateException = (DuplicateJobSubmissionException) executionException.getCause();
    assertTrue(duplicateException.isGloballyTerminated());
}

Example 33 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class DispatcherCleanupITCase method testCleanupAfterLeadershipChange.

@Test
public void testCleanupAfterLeadershipChange() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final JobID jobId = jobGraph.getJobID();
    // Construct job graph store.
    final AtomicInteger actualGlobalCleanupCallCount = new AtomicInteger();
    final OneShotLatch successfulCleanupLatch = new OneShotLatch();
    final RuntimeException temporaryError = new RuntimeException("Unable to remove job graph.");
    final JobGraphStore jobGraphStore = createAndStartJobGraphStoreWithCleanupFailures(1, temporaryError, actualGlobalCleanupCallCount, successfulCleanupLatch);
    haServices.setJobGraphStore(jobGraphStore);
    // Construct leader election service.
    final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
    haServices.setJobMasterLeaderElectionService(jobId, leaderElectionService);
    // start the dispatcher with no retries on cleanup
    final CountDownLatch jobGraphRemovalErrorReceived = new CountDownLatch(1);
    final Dispatcher dispatcher = createTestingDispatcherBuilder().setFatalErrorHandler(throwable -> {
        final Optional<Throwable> maybeError = ExceptionUtils.findThrowable(throwable, temporaryError::equals);
        if (maybeError.isPresent()) {
            jobGraphRemovalErrorReceived.countDown();
        } else {
            testingFatalErrorHandlerResource.getFatalErrorHandler().onFatalError(throwable);
        }
    }).build();
    dispatcher.start();
    toTerminate.add(dispatcher);
    leaderElectionService.isLeader(UUID.randomUUID());
    final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);
    dispatcherGateway.submitJob(jobGraph, TIMEOUT).get();
    waitForJobToFinish(leaderElectionService, dispatcherGateway, jobId);
    jobGraphRemovalErrorReceived.await();
    // Remove job master leadership.
    leaderElectionService.notLeader();
    // This will clear internal state of election service, so a new contender can register.
    leaderElectionService.stop();
    assertThat(successfulCleanupLatch.isTriggered(), CoreMatchers.is(false));
    assertThat("The JobGraph is still stored in the JobGraphStore.", haServices.getJobGraphStore().getJobIds(), CoreMatchers.is(Collections.singleton(jobId)));
    assertThat("The JobResultStore has this job marked as dirty.", haServices.getJobResultStore().getDirtyResults().stream().map(JobResult::getJobId).collect(Collectors.toSet()), CoreMatchers.is(Collections.singleton(jobId)));
    // Run a second dispatcher, that restores our finished job.
    final Dispatcher secondDispatcher = createTestingDispatcherBuilder().setRecoveredDirtyJobs(haServices.getJobResultStore().getDirtyResults()).build();
    secondDispatcher.start();
    toTerminate.add(secondDispatcher);
    leaderElectionService.isLeader(UUID.randomUUID());
    CommonTestUtils.waitUntilCondition(() -> haServices.getJobResultStore().getDirtyResults().isEmpty(), Deadline.fromNow(TimeUtils.toDuration(TIMEOUT)));
    assertThat("The JobGraph is not stored in the JobGraphStore.", haServices.getJobGraphStore().getJobIds(), IsEmptyCollection.empty());
    assertTrue("The JobResultStore has the job listed as clean.", haServices.getJobResultStore().hasJobResultEntry(jobId));
    // wait for the successful cleanup to be triggered
    successfulCleanupLatch.await();
    assertThat(actualGlobalCleanupCallCount.get(), equalTo(2));
}

Example 34 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class FileSystemJobResultStoreTestInternal method testJobResultSerializationDeserialization.

@Test
public void testJobResultSerializationDeserialization() throws IOException {
    fileSystemJobResultStore.createDirtyResult(DUMMY_JOB_RESULT_ENTRY);
    final File dirtyFile = expectedDirtyFile(DUMMY_JOB_RESULT_ENTRY);
    final FileSystemJobResultStore.JsonJobResultEntry deserializedEntry = MAPPER.readValue(dirtyFile, FileSystemJobResultStore.JsonJobResultEntry.class);
    final JobResult deserializedJobResult = deserializedEntry.getJobResult();
    assertThat(deserializedJobResult).extracting(JobResult::getJobId).isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobId());
    assertThat(deserializedJobResult).extracting(JobResult::getApplicationStatus).isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobResult().getApplicationStatus());
    assertThat(deserializedJobResult).extracting(JobResult::getNetRuntime).isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobResult().getNetRuntime());
    assertThat(deserializedJobResult).extracting(JobResult::getSerializedThrowable).isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobResult().getSerializedThrowable());
    assertThat(deserializedJobResult).extracting(JobResult::getAccumulatorResults).isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobResult().getAccumulatorResults());
}

Example 35 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class JobResultStoreContractTest method testGetDirtyResultsWithDirtyEntry.

@Test
default void testGetDirtyResultsWithDirtyEntry() throws IOException {
    JobResultStore jobResultStore = createJobResultStore();
    jobResultStore.createDirtyResult(DUMMY_JOB_RESULT_ENTRY);
    assertThat(jobResultStore.getDirtyResults().stream().map(JobResult::getJobId).collect(Collectors.toList())).singleElement().isEqualTo(DUMMY_JOB_RESULT_ENTRY.getJobId());
}