Examples with TestingHighAvailabilityServicesBuilder org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder used on opensource projects

Example 1 with TestingHighAvailabilityServicesBuilder

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder in project flink by apache.

the class MiniDispatcherTest method setup.

@Before
public void setup() throws Exception {
    highAvailabilityServices = new TestingHighAvailabilityServicesBuilder().build();
    testingJobManagerRunnerFactory = new TestingJobMasterServiceLeadershipRunnerFactory();
    testingCleanupRunnerFactory = new TestingCleanupRunnerFactory();
}

Example 2 with TestingHighAvailabilityServicesBuilder

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder in project flink by apache.

the class ClusterEntrypointTest method testCloseAsyncShouldNotCleanUpHAData.

@Test
public void testCloseAsyncShouldNotCleanUpHAData() throws Exception {
    final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
    final CompletableFuture<Void> closeAndCleanupAllDataFuture = new CompletableFuture<>();
    final HighAvailabilityServices testingHaService = new TestingHighAvailabilityServicesBuilder().setCloseFuture(closeFuture).setCloseAndCleanupAllDataFuture(closeAndCleanupAllDataFuture).build();
    final TestingEntryPoint testingEntryPoint = new TestingEntryPoint.Builder().setConfiguration(flinkConfig).setHighAvailabilityServices(testingHaService).build();
    final CompletableFuture<ApplicationStatus> appStatusFuture = startClusterEntrypoint(testingEntryPoint);
    testingEntryPoint.closeAsync();
    assertThat(appStatusFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS), is(ApplicationStatus.UNKNOWN));
    assertThat(closeFuture.isDone(), is(true));
    assertThat(closeAndCleanupAllDataFuture.isDone(), is(false));
}

Example 3 with TestingHighAvailabilityServicesBuilder

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder in project flink by apache.

the class JobDispatcherLeaderProcessFactoryFactoryTest method createDispatcherLeaderProcessFactoryFromTestInstance.

private static JobDispatcherLeaderProcessFactory createDispatcherLeaderProcessFactoryFromTestInstance(@Nullable JobGraph jobGraph, @Nullable JobResult dirtyJobResult, Path storageDir) throws IOException {
    final JobDispatcherLeaderProcessFactoryFactory testInstance = new JobDispatcherLeaderProcessFactoryFactory(ignoredConfig -> jobGraph);
    final TestingJobResultStore jobResultStore = TestingJobResultStore.builder().withGetDirtyResultsSupplier(() -> CollectionUtil.ofNullable(dirtyJobResult)).build();
    final JobGraphStore jobGraphStore = new StandaloneJobGraphStore();
    return testInstance.createFactory(new TestingJobPersistenceComponentFactory(jobGraphStore, jobResultStore), Executors.directExecutor(), new TestingRpcService(), TestingPartialDispatcherServices.builder().withHighAvailabilityServices(new TestingHighAvailabilityServicesBuilder().setJobGraphStore(jobGraphStore).setJobResultStore(jobResultStore).build()).build(storageDir.toFile(), new Configuration()), NoOpFatalErrorHandler.INSTANCE);
}

Example 4 with TestingHighAvailabilityServicesBuilder

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder in project flink by apache.

the class DefaultJobLeaderServiceTest method removeJobWithFailingLeaderRetrievalServiceStopWillStopListeningToLeaderNotifications.

@Test
public void removeJobWithFailingLeaderRetrievalServiceStopWillStopListeningToLeaderNotifications() throws Exception {
    final FailingSettableLeaderRetrievalService leaderRetrievalService = new FailingSettableLeaderRetrievalService();
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setJobMasterLeaderRetrieverFunction(ignored -> leaderRetrievalService).build();
    final JobID jobId = new JobID();
    final CompletableFuture<JobID> newLeaderFuture = new CompletableFuture<>();
    final TestingJobLeaderListener testingJobLeaderListener = new TestingJobLeaderListener(newLeaderFuture::complete);
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().build();
    rpcServiceResource.getTestingRpcService().registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final JobLeaderService jobLeaderService = createAndStartJobLeaderService(haServices, testingJobLeaderListener);
    try {
        jobLeaderService.addJob(jobId, "foobar");
        jobLeaderService.removeJob(jobId);
        leaderRetrievalService.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        try {
            newLeaderFuture.get(10, TimeUnit.MILLISECONDS);
            fail("The leader future should not be completed.");
        } catch (TimeoutException expected) {
        }
    } finally {
        jobLeaderService.stop();
    }
}

Example 5 with TestingHighAvailabilityServicesBuilder

use of org.apache.flink.runtime.highavailability.TestingHighAvailabilityServicesBuilder in project flink by apache.

the class TaskExecutorSlotLifetimeTest method testUserCodeClassLoaderIsBoundToSlot.

/**
 * Tests that the user code class loader is bound to the lifetime of the slot. This means that
 * it is being reused across a failover, for example. See FLINK-16408.
 */
@Test
public void testUserCodeClassLoaderIsBoundToSlot() throws Exception {
    final Configuration configuration = new Configuration();
    final TestingRpcService rpcService = TESTING_RPC_SERVICE_RESOURCE.getTestingRpcService();
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    final CompletableFuture<SlotReport> firstSlotReportFuture = new CompletableFuture<>();
    resourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        firstSlotReportFuture.complete(resourceIDInstanceIDSlotReportTuple3.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final BlockingQueue<TaskExecutionState> taskExecutionStates = new ArrayBlockingQueue<>(3);
    final OneShotLatch slotsOfferedLatch = new OneShotLatch();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        slotsOfferedLatch.trigger();
        return CompletableFuture.completedFuture(slotOffers);
    }).setUpdateTaskExecutionStateFunction(FunctionUtils.uncheckedFunction(taskExecutionState -> {
        taskExecutionStates.put(taskExecutionState);
        return CompletableFuture.completedFuture(Acknowledge.get());
    })).build();
    final LeaderRetrievalService resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
    final LeaderRetrievalService jobMasterLeaderRetriever = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever).setJobMasterLeaderRetrieverFunction(ignored -> jobMasterLeaderRetriever).build();
    rpcService.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    try (final TaskExecutor taskExecutor = createTaskExecutor(configuration, rpcService, haServices, unresolvedTaskManagerLocation)) {
        taskExecutor.start();
        final SlotReport slotReport = firstSlotReportFuture.join();
        final SlotID firstSlotId = slotReport.iterator().next().getSlotID();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final JobID jobId = new JobID();
        final AllocationID allocationId = new AllocationID();
        taskExecutorGateway.requestSlot(firstSlotId, jobId, allocationId, ResourceProfile.ZERO, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final TaskDeploymentDescriptor tdd = TaskDeploymentDescriptorBuilder.newBuilder(jobId, UserClassLoaderExtractingInvokable.class).setAllocationId(allocationId).build();
        slotsOfferedLatch.await();
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader firstClassLoader = UserClassLoaderExtractingInvokable.take();
        // wait for the first task to finish
        TaskExecutionState taskExecutionState;
        do {
            taskExecutionState = taskExecutionStates.take();
        } while (!taskExecutionState.getExecutionState().isTerminal());
        // check that a second task will re-use the same class loader
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader secondClassLoader = UserClassLoaderExtractingInvokable.take();
        assertThat(firstClassLoader, sameInstance(secondClassLoader));
    }
}