Examples with TestingTaskExecutorGatewayBuilder org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder used on opensource projects

Example 41 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReturnIdleSlotsAfterTimeout.

@Test
public void testReturnIdleSlotsAfterTimeout() {
    final Time idleSlotTimeout = Time.seconds(10);
    final long offerTime = 0;
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().setIdleSlotTimeout(idleSlotTimeout).build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> acceptedSlots = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, new LocalTaskManagerLocation(), testingTaskExecutorGateway);
    // decrease the resource requirements so that slots are no longer needed
    slotPool.decreaseResourceRequirementsBy(resourceRequirements);
    slotPool.releaseIdleSlots(offerTime + idleSlotTimeout.toMilliseconds());
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(acceptedSlots, is(not(empty())));
    assertThat(freedSlots, containsInAnyOrder(acceptedSlots.stream().map(SlotOffer::getAllocationId).toArray()));
    assertNoAvailableAndRequiredResources(slotPool);
}

Example 42 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotsReturnsSlot.

@Test
public void testReleaseSlotsReturnsSlot() {
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().build();
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Collection<SlotOffer> slotOffers = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, taskManagerLocation, testingTaskExecutorGateway);
    slotPool.releaseSlots(taskManagerLocation.getResourceID(), new FlinkException("Test failure"));
    final Collection<AllocationID> freedSlots = freeSlotConsumer.drainFreedSlots();
    assertThat(freedSlots, containsInAnyOrder(slotOffers.stream().map(SlotOffer::getAllocationId).toArray()));
}

Example 43 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testAllocationUpdatesIgnoredIfTaskExecutorUnregistered.

// ---------------------------------------------------------------------------------------------
// Allocation update
// ---------------------------------------------------------------------------------------------
/**
 * Verify that the ack of request slot form unregistered task manager will not cause system
 * breakdown.
 */
@Test
public void testAllocationUpdatesIgnoredIfTaskExecutorUnregistered() throws Exception {
    final CompletableFuture<Acknowledge> slotRequestFuture = new CompletableFuture<>();
    final CompletableFuture<Void> slotRequestCallFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> {
        slotRequestCallFuture.complete(null);
        return slotRequestFuture;
    }).createTestingTaskExecutorGateway();
    // The fatal error handler will exit the system if there is any exceptions in handling the
    // ack of request slot. We need the security manager to verify that would not happen.
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    final JobID jobId = new JobID();
    final ResourceID taskExecutorResourceId = ResourceID.generate();
    final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(taskExecutorResourceId, taskExecutorGateway);
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().processResourceRequirements(createResourceRequirements(jobId, 1));
                    getSlotManager().registerTaskManager(taskExecutionConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                });
                assertFutureCompleteAndReturn(slotRequestCallFuture);
                runInMainThread(() -> {
                    getSlotManager().unregisterTaskManager(taskExecutionConnection.getInstanceID(), TEST_EXCEPTION);
                    slotRequestFuture.complete(Acknowledge.get());
                });
                assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
            });
        }
    };
    System.setSecurityManager(null);
}

Example 44 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testSlotRequestFailure.

// ---------------------------------------------------------------------------------------------
// Slot allocation failure handling
// ---------------------------------------------------------------------------------------------
/**
 * Tests that the SlotManager retries allocating a slot if the TaskExecutor#requestSlot call
 * fails.
 */
@Test
public void testSlotRequestFailure() throws Exception {
    final JobID jobId = new JobID();
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot(jobId);
    final CompletableFuture<Acknowledge> slotRequestFuture1 = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> slotRequestFuture2 = CompletableFuture.completedFuture(Acknowledge.get());
    final Iterator<CompletableFuture<Acknowledge>> slotRequestFutureIterator = Arrays.asList(slotRequestFuture1, slotRequestFuture2).iterator();
    final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(FunctionUtils.uncheckedFunction(requestSlotParameters -> {
        allocationIds.put(requestSlotParameters.f2);
        return slotRequestFutureIterator.next();
    })).createTestingTaskExecutorGateway();
    final ResourceID resourceId = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                    getSlotManager().processResourceRequirements(resourceRequirements);
                });
                final AllocationID firstAllocationId = allocationIds.take();
                assertThat(allocationIds, is(empty()));
                // let the first attempt fail --> this should trigger a second attempt
                runInMainThread(() -> slotRequestFuture1.completeExceptionally(new SlotAllocationException("Test exception.")));
                final AllocationID secondAllocationId = allocationIds.take();
                assertThat(allocationIds, is(empty()));
                final TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(secondAllocationId).get();
                assertEquals(jobId, slot.getJobId());
                assertFalse(getTaskManagerTracker().getAllocatedOrPendingSlot(firstAllocationId).isPresent());
            });
        }
    };
}

Example 45 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testRequirementDeclaration.

private void testRequirementDeclaration(RequirementDeclarationScenario scenario) throws Exception {
    final ResourceID resourceID = ResourceID.generate();
    final JobID jobId = new JobID();
    final SlotID slotId = SlotID.getDynamicSlotID(resourceID);
    final String targetAddress = "localhost";
    final ResourceRequirements requirements = ResourceRequirements.create(jobId, targetAddress, Collections.singleton(ResourceRequirement.create(DEFAULT_SLOT_RESOURCE_PROFILE, 1)));
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    // accept an incoming slot request
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(tuple6);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    new Context() {

        {
            runTest(() -> {
                if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_BEFORE_REQUIREMENT_DECLARATION) {
                    runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                }
                runInMainThread(() -> getSlotManager().processResourceRequirements(requirements));
                if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_AFTER_REQUIREMENT_DECLARATION) {
                    runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                }
                assertThat(assertFutureCompleteAndReturn(requestFuture), is(equalTo(Tuple6.of(slotId, jobId, assertFutureCompleteAndReturn(requestFuture).f2, DEFAULT_SLOT_RESOURCE_PROFILE, targetAddress, getResourceManagerId()))));
                final TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(requestFuture).f2).get();
                assertEquals("The slot has not been allocated to the expected allocation id.", assertFutureCompleteAndReturn(requestFuture).f2, slot.getAllocationId());
            });
        }
    };
}