Examples with JobManagerRunner org.apache.flink.runtime.jobmaster.JobManagerRunner used on opensource projects

Example 1 with JobManagerRunner

use of org.apache.flink.runtime.jobmaster.JobManagerRunner in project flink by apache.

the class MiniClusterJobDispatcher method shutdown.

// ------------------------------------------------------------------------
//  life cycle
// ------------------------------------------------------------------------
/**
	 * Shuts down the mini cluster dispatcher. If a job is currently running, that job will be
	 * terminally failed.
	 */
public void shutdown() {
    synchronized (lock) {
        if (!shutdown) {
            shutdown = true;
            LOG.info("Shutting down the job dispatcher");
            // in this shutdown code we copy the references to the stack first,
            // to avoid concurrent modification
            JobManagerRunner[] runners = this.runners;
            if (runners != null) {
                this.runners = null;
                for (JobManagerRunner runner : runners) {
                    runner.shutdown();
                }
            }
        }
    }
}

Example 2 with JobManagerRunner

use of org.apache.flink.runtime.jobmaster.JobManagerRunner in project flink by apache.

the class DispatcherCleanupITCase method testCleanupNotCancellable.

@Test
public void testCleanupNotCancellable() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final JobID jobId = jobGraph.getJobID();
    final JobResultStore jobResultStore = new EmbeddedJobResultStore();
    jobResultStore.createDirtyResult(new JobResultEntry(TestingJobResultStore.createSuccessfulJobResult(jobId)));
    haServices.setJobResultStore(jobResultStore);
    // Instantiates JobManagerRunner
    final CompletableFuture<Void> jobManagerRunnerCleanupFuture = new CompletableFuture<>();
    final AtomicReference<JobManagerRunner> jobManagerRunnerEntry = new AtomicReference<>();
    final JobManagerRunnerRegistry jobManagerRunnerRegistry = TestingJobManagerRunnerRegistry.newSingleJobBuilder(jobManagerRunnerEntry).withLocalCleanupAsyncFunction((actualJobId, executor) -> jobManagerRunnerCleanupFuture).build();
    final Dispatcher dispatcher = createTestingDispatcherBuilder().setJobManagerRunnerRegistry(jobManagerRunnerRegistry).build();
    dispatcher.start();
    toTerminate.add(dispatcher);
    CommonTestUtils.waitUntilCondition(() -> jobManagerRunnerEntry.get() != null, Deadline.fromNow(Duration.ofSeconds(10)), "JobManagerRunner wasn't loaded in time.");
    assertThat("The JobResultStore should have this job still marked as dirty.", haServices.getJobResultStore().hasDirtyJobResultEntry(jobId), CoreMatchers.is(true));
    final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);
    try {
        dispatcherGateway.cancelJob(jobId, TIMEOUT).get();
        Assert.fail("Should fail because cancelling the cleanup is not allowed.");
    } catch (ExecutionException e) {
        assertThat(e, FlinkMatchers.containsCause(JobCancellationFailedException.class));
    }
    jobManagerRunnerCleanupFuture.complete(null);
    CommonTestUtils.waitUntilCondition(() -> haServices.getJobResultStore().hasCleanJobResultEntry(jobId), Deadline.fromNow(Duration.ofSeconds(60)), "The JobResultStore should have this job marked as clean now.");
}

Example 3 with JobManagerRunner

use of org.apache.flink.runtime.jobmaster.JobManagerRunner in project flink by apache.

the class DispatcherCleanupITCase method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    haServices.setCheckpointRecoveryFactory(new PerJobCheckpointRecoveryFactory<EmbeddedCompletedCheckpointStore>((maxCheckpoints, previous, sharedStateRegistryFactory, ioExecutor) -> {
        if (previous != null) {
            // First job cleanup still succeeded for the
            // CompletedCheckpointStore because the JobGraph cleanup happens
            // after the JobManagerRunner closing
            assertTrue(previous.getShutdownStatus().isPresent());
            assertTrue(previous.getAllCheckpoints().isEmpty());
            return new EmbeddedCompletedCheckpointStore(maxCheckpoints, previous.getAllCheckpoints(), sharedStateRegistryFactory.create(ioExecutor, previous.getAllCheckpoints()));
        }
        return new EmbeddedCompletedCheckpointStore(maxCheckpoints, Collections.emptyList(), sharedStateRegistryFactory.create(ioExecutor, Collections.emptyList()));
    }));
}

Example 4 with JobManagerRunner

use of org.apache.flink.runtime.jobmaster.JobManagerRunner in project flink by apache.

the class DispatcherResourceCleanupTest method testGlobalCleanupWhenJobFinishedWhileClosingDispatcher.

@Test
public void testGlobalCleanupWhenJobFinishedWhileClosingDispatcher() throws Exception {
    final TestingJobManagerRunner testingJobManagerRunner = TestingJobManagerRunner.newBuilder().setBlockingTermination(true).setJobId(jobId).build();
    final Queue<JobManagerRunner> jobManagerRunners = new ArrayDeque<>(Arrays.asList(testingJobManagerRunner));
    startDispatcher(new QueueJobManagerRunnerFactory(jobManagerRunners));
    submitJobAndWait();
    final CompletableFuture<Void> dispatcherTerminationFuture = dispatcher.closeAsync();
    testingJobManagerRunner.getCloseAsyncCalledLatch().await();
    testingJobManagerRunner.completeResultFuture(new ExecutionGraphInfo(new ArchivedExecutionGraphBuilder().setJobID(jobId).setState(JobStatus.FINISHED).build()));
    testingJobManagerRunner.completeTerminationFuture();
    // check that no exceptions have been thrown
    dispatcherTerminationFuture.get();
    assertGlobalCleanupTriggered(jobId);
}

Example 5 with JobManagerRunner

use of org.apache.flink.runtime.jobmaster.JobManagerRunner in project flink by apache.

the class Dispatcher method terminateJob.

private void terminateJob(JobID jobId) {
    if (jobManagerRunnerRegistry.isRegistered(jobId)) {
        final JobManagerRunner jobManagerRunner = jobManagerRunnerRegistry.get(jobId);
        jobManagerRunner.closeAsync();
    }
}