Examples with TestingTaskExecutorGateway org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway used on opensource projects

Example 21 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class JobMasterTest method runJobFailureWhenTaskExecutorTerminatesTest.

private void runJobFailureWhenTaskExecutorTerminatesTest(HeartbeatServices heartbeatServices, BiConsumer<LocalUnresolvedTaskManagerLocation, JobMasterGateway> jobReachedRunningState, BiFunction<JobMasterGateway, ResourceID, BiFunction<ResourceID, AllocatedSlotReport, CompletableFuture<Void>>> heartbeatConsumerFunction) throws Exception {
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    final JobMasterBuilder.TestingOnCompletionActions onCompletionActions = new JobMasterBuilder.TestingOnCompletionActions();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withResourceId(jmResourceId).withHighAvailabilityServices(haServices).withHeartbeatServices(heartbeatServices).withOnCompletionActions(onCompletionActions).createJobMaster();
    try {
        jobMaster.start();
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        final LocalUnresolvedTaskManagerLocation taskManagerUnresolvedLocation = new LocalUnresolvedTaskManagerLocation();
        final CompletableFuture<ExecutionAttemptID> taskDeploymentFuture = new CompletableFuture<>();
        final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
            taskDeploymentFuture.complete(taskDeploymentDescriptor.getExecutionAttemptId());
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).setHeartbeatJobManagerFunction(heartbeatConsumerFunction.apply(jobMasterGateway, taskManagerUnresolvedLocation.getResourceID())).createTestingTaskExecutorGateway();
        final Collection<SlotOffer> slotOffers = registerSlotsAtJobMaster(1, jobMasterGateway, jobGraph.getJobID(), taskExecutorGateway, taskManagerUnresolvedLocation);
        assertThat(slotOffers, hasSize(1));
        final ExecutionAttemptID executionAttemptId = taskDeploymentFuture.get();
        jobMasterGateway.updateTaskExecutionState(new TaskExecutionState(executionAttemptId, ExecutionState.INITIALIZING)).get();
        jobMasterGateway.updateTaskExecutionState(new TaskExecutionState(executionAttemptId, ExecutionState.RUNNING)).get();
        jobReachedRunningState.accept(taskManagerUnresolvedLocation, jobMasterGateway);
        final ArchivedExecutionGraph archivedExecutionGraph = onCompletionActions.getJobReachedGloballyTerminalStateFuture().get().getArchivedExecutionGraph();
        assertThat(archivedExecutionGraph.getState(), is(JobStatus.FAILED));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 22 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class JobMasterTest method testJobMasterAcknowledgesDuplicateTaskExecutorRegistrations.

@Test
public void testJobMasterAcknowledgesDuplicateTaskExecutorRegistrations() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
    rpcService.registerGateway(testingTaskExecutorGateway.getAddress(), testingTaskExecutorGateway);
    try {
        jobMaster.start();
        final TaskManagerRegistrationInformation taskManagerRegistrationInformation = TaskManagerRegistrationInformation.create(testingTaskExecutorGateway.getAddress(), new LocalUnresolvedTaskManagerLocation(), UUID.randomUUID());
        final CompletableFuture<RegistrationResponse> firstRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), taskManagerRegistrationInformation, testingTimeout);
        final CompletableFuture<RegistrationResponse> secondRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), taskManagerRegistrationInformation, testingTimeout);
        assertThat(firstRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        assertThat(secondRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 23 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class JobMasterTest method testAllocatedSlotReportDoesNotContainStaleInformation.

/**
 * Tests that the {@link AllocatedSlotReport} contains up to date information and not stale
 * information about the allocated slots on the {@link JobMaster}.
 *
 * <p>This is a probabilistic test case which only fails if executed repeatedly without the fix
 * for FLINK-12863.
 */
@Test
public void testAllocatedSlotReportDoesNotContainStaleInformation() throws Exception {
    final CompletableFuture<Void> assertionFuture = new CompletableFuture<>();
    final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    final AtomicBoolean terminateHeartbeatVerification = new AtomicBoolean(false);
    final OneShotLatch hasReceivedSlotOffers = new OneShotLatch();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setHeartbeatJobManagerFunction((taskManagerId, allocatedSlotReport) -> {
        try {
            if (hasReceivedSlotOffers.isTriggered()) {
                assertThat(allocatedSlotReport.getAllocatedSlotInfos(), hasSize(1));
            } else {
                assertThat(allocatedSlotReport.getAllocatedSlotInfos(), empty());
            }
        } catch (AssertionError e) {
            assertionFuture.completeExceptionally(e);
        }
        if (terminateHeartbeatVerification.get()) {
            assertionFuture.complete(null);
        }
        return FutureUtils.completedVoidFuture();
    }).createTestingTaskExecutorGateway();
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    final JobManagerSharedServices jobManagerSharedServices = new TestingJobManagerSharedServicesBuilder().build();
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withHeartbeatServices(new HeartbeatServices(5L, 1000L)).withSlotPoolServiceSchedulerFactory(DefaultSlotPoolServiceSchedulerFactory.create(new TestingSlotPoolFactory(hasReceivedSlotOffers), new DefaultSchedulerFactory())).createJobMaster();
    jobMaster.start();
    try {
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        // register task manager will trigger monitor heartbeat target, schedule heartbeat
        // request at interval time
        CompletableFuture<RegistrationResponse> registrationResponse = jobMasterGateway.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(taskExecutorGateway.getAddress(), unresolvedTaskManagerLocation, TestingUtils.zeroUUID()), testingTimeout);
        // wait for the completion of the registration
        registrationResponse.get();
        final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
        final CompletableFuture<Collection<SlotOffer>> slotOfferFuture = jobMasterGateway.offerSlots(unresolvedTaskManagerLocation.getResourceID(), Collections.singleton(slotOffer), testingTimeout);
        assertThat(slotOfferFuture.get(), containsInAnyOrder(slotOffer));
        terminateHeartbeatVerification.set(true);
        // make sure that no assertion has been violated
        assertionFuture.get();
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
        jobManagerSharedServices.shutdown();
    }
}

Example 24 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReturnIdleSlotsAfterTimeout.

@Test
public void testReturnIdleSlotsAfterTimeout() {
    final Time idleSlotTimeout = Time.seconds(10);
    final long offerTime = 0;
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().setIdleSlotTimeout(idleSlotTimeout).build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> acceptedSlots = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, new LocalTaskManagerLocation(), testingTaskExecutorGateway);
    // decrease the resource requirements so that slots are no longer needed
    slotPool.decreaseResourceRequirementsBy(resourceRequirements);
    slotPool.releaseIdleSlots(offerTime + idleSlotTimeout.toMilliseconds());
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(acceptedSlots, is(not(empty())));
    assertThat(freedSlots, containsInAnyOrder(acceptedSlots.stream().map(SlotOffer::getAllocationId).toArray()));
    assertNoAvailableAndRequiredResources(slotPool);
}

Example 25 with TestingTaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotsReturnsSlot.

@Test
public void testReleaseSlotsReturnsSlot() {
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> slotOffers = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, taskManagerLocation, testingTaskExecutorGateway);
    slotPool.releaseSlots(taskManagerLocation.getResourceID(), new FlinkException("Test failure"));
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(freedSlots, containsInAnyOrder(slotOffers.stream().map(SlotOffer::getAllocationId).toArray()));
}