Examples with TestingTaskExecutorGatewayBuilder org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder used on opensource projects

Example 6 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DeclarativeSlotPoolServiceTest method testSlotOfferingOfUnknownTaskManagerIsIgnored.

@Test
public void testSlotOfferingOfUnknownTaskManagerIsIgnored() throws Exception {
    try (DeclarativeSlotPoolService declarativeSlotPoolService = createDeclarativeSlotPoolService()) {
        final Collection<SlotOffer> slotOffers = Collections.singletonList(new SlotOffer(new AllocationID(), 0, ResourceProfile.UNKNOWN));
        final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
        final Collection<SlotOffer> acceptedSlots = declarativeSlotPoolService.offerSlots(taskManagerLocation, new RpcTaskManagerGateway(new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway(), jobMasterId), slotOffers);
        assertThat(acceptedSlots, is(empty()));
    }
}

Example 7 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method withSlotPoolContainingOneTaskManagerWithTwoSlotsWithUniqueResourceProfiles.

private static void withSlotPoolContainingOneTaskManagerWithTwoSlotsWithUniqueResourceProfiles(QuadConsumer<DefaultDeclarativeSlotPool, SlotOffer, SlotOffer, TaskManagerLocation> test) {
    final DefaultDeclarativeSlotPool slotPool = DefaultDeclarativeSlotPoolBuilder.builder().build();
    final ResourceCounter resourceRequirements = ResourceCounter.withResource(RESOURCE_PROFILE_1, 1).add(RESOURCE_PROFILE_2, 1);
    final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    final Iterator<SlotOffer> slotOffers = increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, taskManagerLocation, testingTaskExecutorGateway).iterator();
    final SlotOffer slot1 = slotOffers.next();
    final SlotOffer slot2 = slotOffers.next();
    test.accept(slotPool, slot1, slot2, taskManagerLocation);
}

Example 8 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DefaultDeclarativeSlotPoolTest method testReleaseSlotReturnsSlot.

@Test
public void testReleaseSlotReturnsSlot() throws InterruptedException {
    final NewSlotsService notifyNewSlots = new NewSlotsService();
    final DefaultDeclarativeSlotPool slotPool = createDefaultDeclarativeSlotPoolWithNewSlotsListener(notifyNewSlots);
    final ResourceCounter resourceRequirements = createResourceRequirements();
    final FreeSlotConsumer freeSlotConsumer = new FreeSlotConsumer();
    final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setFreeSlotFunction(freeSlotConsumer).createTestingTaskExecutorGateway();
    increaseRequirementsAndOfferSlotsToSlotPool(slotPool, resourceRequirements, new LocalTaskManagerLocation(), testingTaskExecutorGateway);
    final Collection<? extends PhysicalSlot> physicalSlots = notifyNewSlots.takeNewSlots();
    final PhysicalSlot physicalSlot = physicalSlots.iterator().next();
    slotPool.releaseSlot(physicalSlot.getAllocationId(), new FlinkException("Test failure"));
    final AllocationID freedSlot = Iterables.getOnlyElement(freeSlotConsumer.drainFreedSlots());
    assertThat(freedSlot, is(physicalSlot.getAllocationId()));
}

Example 9 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DeclarativeSlotManagerTest method testSlotReportWithConflictingJobIdDuringSlotAllocation.

/**
 * Tests that a pending slot allocation is cancelled if a slot report indicates that the slot is
 * already allocated by another job.
 */
@Test
public void testSlotReportWithConflictingJobIdDuringSlotAllocation() throws Exception {
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    final ArrayBlockingQueue<SlotID> requestedSlotIds = new ArrayBlockingQueue<>(2);
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(FunctionUtils.uncheckedFunction(requestSlotParameters -> {
        requestedSlotIds.put(requestSlotParameters.f0);
        return new CompletableFuture<>();
    })).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection(taskExecutorGateway);
    final ResourceID resourceId = taskExecutorConnection.getResourceID();
    final SlotID slotId1 = new SlotID(resourceId, 0);
    final SlotID slotId2 = new SlotID(resourceId, 1);
    final SlotReport slotReport = new SlotReport(Arrays.asList(createFreeSlotStatus(slotId1), createFreeSlotStatus(slotId2)));
    final ScheduledExecutor mainThreadExecutor = new ManuallyTriggeredScheduledExecutor();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setScheduledExecutor(mainThreadExecutor).build()) {
        slotManager.start(ResourceManagerId.generate(), mainThreadExecutor, new TestingResourceActionsBuilder().build());
        slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.processResourceRequirements(resourceRequirements);
        final SlotID firstRequestedSlotId = requestedSlotIds.take();
        final SlotID freeSlotId = firstRequestedSlotId.equals(slotId1) ? slotId2 : slotId1;
        final SlotReport newSlotReport = new SlotReport(Arrays.asList(createAllocatedSlotStatus(firstRequestedSlotId), createFreeSlotStatus(freeSlotId)));
        slotManager.reportSlotStatus(taskExecutorConnection.getInstanceID(), newSlotReport);
        final SlotID secondRequestedSlotId = requestedSlotIds.take();
        assertEquals(freeSlotId, secondRequestedSlotId);
    }
}

Example 10 with TestingTaskExecutorGatewayBuilder

use of org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder in project flink by apache.

the class DeclarativeSlotManagerTest method testRequirementDeclaration.

private void testRequirementDeclaration(RequirementDeclarationScenario scenario) throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final ResourceID resourceID = ResourceID.generate();
    final JobID jobId = new JobID();
    final SlotID slotId = new SlotID(resourceID, 0);
    final String targetAddress = "localhost";
    final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    // accept an incoming slot request
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(Tuple6.of(tuple6.f0, tuple6.f1, tuple6.f2, tuple6.f3, tuple6.f4, tuple6.f5));
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
    final SlotReport slotReport = new SlotReport(slotStatus);
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    try (DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStartWithDirectExec(resourceManagerId, new TestingResourceActionsBuilder().build())) {
        if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_BEFORE_REQUIREMENT_DECLARATION) {
            slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        }
        final ResourceRequirements requirements = ResourceRequirements.create(jobId, targetAddress, Collections.singleton(ResourceRequirement.create(resourceProfile, 1)));
        slotManager.processResourceRequirements(requirements);
        if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_AFTER_REQUIREMENT_DECLARATION) {
            slotManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        }
        assertThat(requestFuture.get(), is(equalTo(Tuple6.of(slotId, jobId, requestFuture.get().f2, resourceProfile, targetAddress, resourceManagerId))));
        DeclarativeTaskManagerSlot slot = slotTracker.getSlot(slotId);
        assertEquals("The slot has not been allocated to the expected allocation id.", jobId, slot.getJobId());
    }
}