use of org.apache.flink.runtime.jobmanager.JobGraphStore in project flink by apache.
the class SessionDispatcherLeaderProcessTest method onRemovedJobGraph_terminatesRunningJob.
@Test
public void onRemovedJobGraph_terminatesRunningJob() throws Exception {
jobGraphStore = TestingJobGraphStore.newBuilder().setInitialJobGraphs(Collections.singleton(JOB_GRAPH)).build();
final CompletableFuture<JobID> terminateJobFuture = new CompletableFuture<>();
final TestingDispatcherGatewayService testingDispatcherService = TestingDispatcherGatewayService.newBuilder().setOnRemovedJobGraphFunction(jobID -> {
terminateJobFuture.complete(jobID);
return FutureUtils.completedVoidFuture();
}).build();
dispatcherServiceFactory = createFactoryBasedOnGenericSupplier(() -> testingDispatcherService);
final ExecutorService executorService = Executors.newSingleThreadExecutor();
try (final SessionDispatcherLeaderProcess dispatcherLeaderProcess = createDispatcherLeaderProcess()) {
dispatcherLeaderProcess.start();
// wait for the dispatcher process to be created
dispatcherLeaderProcess.getDispatcherGateway().get();
// now remove the Job from the JobGraphStore and notify the dispatcher service
jobGraphStore.globalCleanupAsync(JOB_GRAPH.getJobID(), executorService).join();
dispatcherLeaderProcess.onRemovedJobGraph(JOB_GRAPH.getJobID());
assertThat(terminateJobFuture.get()).isEqualTo(JOB_GRAPH.getJobID());
} finally {
assertThat(executorService.shutdownNow()).isEmpty();
}
}
use of org.apache.flink.runtime.jobmanager.JobGraphStore in project flink by apache.
the class ZooKeeperDefaultDispatcherRunnerTest method testResourceCleanupUnderLeadershipChange.
/**
* See FLINK-11665.
*/
@Test
public void testResourceCleanupUnderLeadershipChange() throws Exception {
final TestingRpcService rpcService = testingRpcServiceResource.getTestingRpcService();
final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration, fatalErrorHandler).asCuratorFramework();
try (final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder().setDispatcherLeaderElectionService(dispatcherLeaderElectionService).setJobMasterLeaderRetrieverFunction(jobId -> ZooKeeperUtils.createLeaderRetrievalService(client)).build()) {
final PartialDispatcherServices partialDispatcherServices = new PartialDispatcherServices(configuration, highAvailabilityServices, CompletableFuture::new, blobServer, new TestingHeartbeatServices(), UnregisteredMetricGroups::createUnregisteredJobManagerMetricGroup, new MemoryExecutionGraphInfoStore(), fatalErrorHandler, VoidHistoryServerArchivist.INSTANCE, null, ForkJoinPool.commonPool(), new DispatcherOperationCaches());
final DefaultDispatcherRunnerFactory defaultDispatcherRunnerFactory = DefaultDispatcherRunnerFactory.createSessionRunner(SessionDispatcherFactory.INSTANCE);
try (final DispatcherRunner dispatcherRunner = createDispatcherRunner(rpcService, dispatcherLeaderElectionService, new JobPersistenceComponentFactory() {
@Override
public JobGraphStore createJobGraphStore() {
return createZooKeeperJobGraphStore(client);
}
@Override
public JobResultStore createJobResultStore() {
return new EmbeddedJobResultStore();
}
}, partialDispatcherServices, defaultDispatcherRunnerFactory)) {
// initial run
DispatcherGateway dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);
final JobGraph jobGraph = createJobGraphWithBlobs();
LOG.info("Initial job submission {}.", jobGraph.getJobID());
dispatcherGateway.submitJob(jobGraph, TESTING_TIMEOUT).get();
dispatcherLeaderElectionService.notLeader();
// recovering submitted jobs
LOG.info("Re-grant leadership first time.");
dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);
LOG.info("Cancel recovered job {}.", jobGraph.getJobID());
// cancellation of the job should remove everything
final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway.requestJobResult(jobGraph.getJobID(), TESTING_TIMEOUT);
dispatcherGateway.cancelJob(jobGraph.getJobID(), TESTING_TIMEOUT).get();
// a successful cancellation should eventually remove all job information
final JobResult jobResult = jobResultFuture.get();
assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.CANCELED));
dispatcherLeaderElectionService.notLeader();
// check that the job has been removed from ZooKeeper
final JobGraphStore submittedJobGraphStore = createZooKeeperJobGraphStore(client);
CommonTestUtils.waitUntilCondition(() -> submittedJobGraphStore.getJobIds().isEmpty(), Deadline.fromNow(VERIFICATION_TIMEOUT), 20L);
}
}
// check resource clean up
assertThat(clusterHaStorageDir.listFiles(), is(emptyArray()));
}
Aggregations