Examples with TestingHighAvailabilityServices org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices used on opensource projects

Example 26 with TestingHighAvailabilityServices

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices in project flink by apache.

the class AbstractDispatcherTest method setUp.

@Before
public void setUp() throws Exception {
    heartbeatServices = new HeartbeatServices(1000L, 10000L);
    haServices = new TestingHighAvailabilityServices();
    haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
    haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
    haServices.setJobGraphStore(new StandaloneJobGraphStore());
    haServices.setJobResultStore(new EmbeddedJobResultStore());
    configuration = new Configuration();
    blobServer = new BlobServer(configuration, temporaryFolder.newFolder(), new VoidBlobStore());
}

Example 27 with TestingHighAvailabilityServices

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices in project flink by apache.

the class ZooKeeperDefaultDispatcherRunnerTest method testResourceCleanupUnderLeadershipChange.

/**
 * See FLINK-11665.
 */
@Test
public void testResourceCleanupUnderLeadershipChange() throws Exception {
    final TestingRpcService rpcService = testingRpcServiceResource.getTestingRpcService();
    final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
    final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration, fatalErrorHandler).asCuratorFramework();
    try (final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder().setDispatcherLeaderElectionService(dispatcherLeaderElectionService).setJobMasterLeaderRetrieverFunction(jobId -> ZooKeeperUtils.createLeaderRetrievalService(client)).build()) {
        final PartialDispatcherServices partialDispatcherServices = new PartialDispatcherServices(configuration, highAvailabilityServices, CompletableFuture::new, blobServer, new TestingHeartbeatServices(), UnregisteredMetricGroups::createUnregisteredJobManagerMetricGroup, new MemoryExecutionGraphInfoStore(), fatalErrorHandler, VoidHistoryServerArchivist.INSTANCE, null, ForkJoinPool.commonPool(), new DispatcherOperationCaches());
        final DefaultDispatcherRunnerFactory defaultDispatcherRunnerFactory = DefaultDispatcherRunnerFactory.createSessionRunner(SessionDispatcherFactory.INSTANCE);
        try (final DispatcherRunner dispatcherRunner = createDispatcherRunner(rpcService, dispatcherLeaderElectionService, new JobPersistenceComponentFactory() {

            @Override
            public JobGraphStore createJobGraphStore() {
                return createZooKeeperJobGraphStore(client);
            }

            @Override
            public JobResultStore createJobResultStore() {
                return new EmbeddedJobResultStore();
            }
        }, partialDispatcherServices, defaultDispatcherRunnerFactory)) {
            // initial run
            DispatcherGateway dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);
            final JobGraph jobGraph = createJobGraphWithBlobs();
            LOG.info("Initial job submission {}.", jobGraph.getJobID());
            dispatcherGateway.submitJob(jobGraph, TESTING_TIMEOUT).get();
            dispatcherLeaderElectionService.notLeader();
            // recovering submitted jobs
            LOG.info("Re-grant leadership first time.");
            dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);
            LOG.info("Cancel recovered job {}.", jobGraph.getJobID());
            // cancellation of the job should remove everything
            final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway.requestJobResult(jobGraph.getJobID(), TESTING_TIMEOUT);
            dispatcherGateway.cancelJob(jobGraph.getJobID(), TESTING_TIMEOUT).get();
            // a successful cancellation should eventually remove all job information
            final JobResult jobResult = jobResultFuture.get();
            assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.CANCELED));
            dispatcherLeaderElectionService.notLeader();
            // check that the job has been removed from ZooKeeper
            final JobGraphStore submittedJobGraphStore = createZooKeeperJobGraphStore(client);
            CommonTestUtils.waitUntilCondition(() -> submittedJobGraphStore.getJobIds().isEmpty(), Deadline.fromNow(VERIFICATION_TIMEOUT), 20L);
        }
    }
    // check resource clean up
    assertThat(clusterHaStorageDir.listFiles(), is(emptyArray()));
}

Example 28 with TestingHighAvailabilityServices

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices in project flink by apache.

the class JobMasterTest method setup.

@Before
public void setup() throws IOException {
    configuration = new Configuration();
    haServices = new TestingHighAvailabilityServices();
    jobMasterId = JobMasterId.generate();
    jmResourceId = ResourceID.generate();
    testingFatalErrorHandler = new TestingFatalErrorHandler();
    haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
    rmLeaderRetrievalService = new SettableLeaderRetrievalService(null, null);
    haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
    configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}

Example 29 with TestingHighAvailabilityServices

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices in project flink by apache.

the class DefaultJobLeaderServiceTest method doesNotReconnectAfterTargetLostLeadership.

/**
 * Tests that the JobLeaderService won't try to reconnect to JobMaster after it has lost the
 * leadership. See FLINK-16836.
 */
@Test
public void doesNotReconnectAfterTargetLostLeadership() throws Exception {
    final JobID jobId = new JobID();
    final SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService();
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setJobMasterLeaderRetrieverFunction(ignored -> leaderRetrievalService).build();
    final TestingJobMasterGateway jobMasterGateway = registerJobMaster();
    final OneShotLatch jobManagerGainedLeadership = new OneShotLatch();
    final TestingJobLeaderListener testingJobLeaderListener = new TestingJobLeaderListener(ignored -> jobManagerGainedLeadership.trigger());
    final JobLeaderService jobLeaderService = createAndStartJobLeaderService(haServices, testingJobLeaderListener);
    try {
        jobLeaderService.addJob(jobId, jobMasterGateway.getAddress());
        leaderRetrievalService.notifyListener(jobMasterGateway.getAddress(), UUID.randomUUID());
        jobManagerGainedLeadership.await();
        // revoke the leadership
        leaderRetrievalService.notifyListener(null, null);
        testingJobLeaderListener.waitUntilJobManagerLostLeadership();
        jobLeaderService.reconnect(jobId);
    } finally {
        jobLeaderService.stop();
    }
}

Example 30 with TestingHighAvailabilityServices

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices in project flink by apache.

the class DefaultJobLeaderServiceTest method canReconnectToOldLeaderWithSameLeaderAddress.

/**
 * Tests that the JobLeaderService can reconnect to an old leader which seemed to have lost the
 * leadership in between. See FLINK-14316.
 */
@Test
public void canReconnectToOldLeaderWithSameLeaderAddress() throws Exception {
    final JobID jobId = new JobID();
    final SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService();
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setJobMasterLeaderRetrieverFunction(ignored -> leaderRetrievalService).build();
    final TestingJobMasterGateway jobMasterGateway = registerJobMaster();
    final BlockingQueue<JobID> leadershipQueue = new ArrayBlockingQueue<>(1);
    final TestingJobLeaderListener testingJobLeaderListener = new TestingJobLeaderListener(leadershipQueue::offer);
    final JobLeaderService jobLeaderService = createAndStartJobLeaderService(haServices, testingJobLeaderListener);
    try {
        jobLeaderService.addJob(jobId, jobMasterGateway.getAddress());
        final UUID leaderSessionId = UUID.randomUUID();
        leaderRetrievalService.notifyListener(jobMasterGateway.getAddress(), leaderSessionId);
        // wait for the first leadership
        assertThat(leadershipQueue.take(), is(jobId));
        // revoke the leadership
        leaderRetrievalService.notifyListener(null, null);
        testingJobLeaderListener.waitUntilJobManagerLostLeadership();
        leaderRetrievalService.notifyListener(jobMasterGateway.getAddress(), leaderSessionId);
        // check that we obtain the leadership a second time
        assertThat(leadershipQueue.take(), is(jobId));
    } finally {
        jobLeaderService.stop();
    }
}