Examples with WorkerResourceSpec org.apache.flink.runtime.resourcemanager.WorkerResourceSpec used on opensource projects

Example 11 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation.

// ---------------------------------------------------------------------------------------------
// Requirement declaration
// ---------------------------------------------------------------------------------------------
/**
 * Tests that a requirement declaration with no free slots will trigger the resource allocation.
 */
@Test
public void testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation() throws Exception {
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    final CompletableFuture<WorkerResourceSpec> allocateResourceFuture = new CompletableFuture<>();
    new Context() {

        {
            resourceActionsBuilder.setAllocateResourceConsumer(allocateResourceFuture::complete);
            runTest(() -> {
                runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements));
                assertFutureCompleteAndReturn(allocateResourceFuture);
            });
        }
    };
}

Example 12 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class DeclarativeSlotManagerTest method testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation.

/**
 * Tests that a slot request with no free slots will trigger the resource allocation.
 */
@Test
public void testRequirementDeclarationWithoutFreeSlotsTriggersWorkerAllocation() throws Exception {
    final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
    final ResourceRequirements resourceRequirements = createResourceRequirementsForSingleSlot();
    CompletableFuture<WorkerResourceSpec> allocateResourceFuture = new CompletableFuture<>();
    ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().setAllocateResourceConsumer(allocateResourceFuture::complete).build();
    try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {
        slotManager.processResourceRequirements(resourceRequirements);
        allocateResourceFuture.get();
    }
}

Example 13 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class TaskExecutorManagerTest method testWorkerOnlyAllocatedIfRequestedSlotCouldBeFulfilled.

/**
 * Test that the task executor manager only allocates new workers if their worker spec can
 * fulfill the requested resource profile.
 */
@Test
public void testWorkerOnlyAllocatedIfRequestedSlotCouldBeFulfilled() {
    final int numCoresPerWorker = 1;
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(numCoresPerWorker).build();
    final ResourceProfile requestedProfile = ResourceProfile.newBuilder().setCpuCores(numCoresPerWorker + 1).build();
    final AtomicInteger resourceRequests = new AtomicInteger(0);
    ResourceActions resourceActions = createResourceActionsBuilder().setAllocateResourceFunction(ignored -> {
        resourceRequests.incrementAndGet();
        return true;
    }).build();
    try (final TaskExecutorManager taskExecutorManager = createTaskExecutorManagerBuilder().setDefaultWorkerResourceSpec(workerResourceSpec).setNumSlotsPerWorker(1).setMaxNumSlots(1).setResourceActions(resourceActions).createTaskExecutorManager()) {
        assertThat(taskExecutorManager.allocateWorker(requestedProfile).orElse(null), nullValue());
        assertThat(resourceRequests.get(), is(0));
    }
}

Example 14 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class TaskExecutorManagerTest method testPendingSlotNotFulfilledIfProfilesAreNotExactMatch.

/**
 * Tests that a pending slot is only fulfilled by an exactly matching received slot.
 */
@Test
public void testPendingSlotNotFulfilledIfProfilesAreNotExactMatch() {
    final int numWorkerCpuCores = 3;
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(numWorkerCpuCores).build();
    final ResourceProfile requestedSlotProfile = ResourceProfile.newBuilder().setCpuCores(numWorkerCpuCores).build();
    final ResourceProfile offeredSlotProfile = ResourceProfile.newBuilder().setCpuCores(numWorkerCpuCores - 1).build();
    try (final TaskExecutorManager taskExecutorManager = createTaskExecutorManagerBuilder().setDefaultWorkerResourceSpec(workerResourceSpec).setNumSlotsPerWorker(// set to one so that the slot profiles directly correspond to
    1).setMaxNumSlots(2).createTaskExecutorManager()) {
        // create pending slot
        taskExecutorManager.allocateWorker(requestedSlotProfile);
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
        createAndRegisterTaskExecutor(taskExecutorManager, 1, offeredSlotProfile);
        // the slot from the task executor should be accepted, but we should still be waiting
        // for the originally requested slot
        assertThat(taskExecutorManager.getNumberRegisteredSlots(), is(1));
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
    }
}

Example 15 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class SlotManagerUtilsTest method testGenerateDefaultSlotProfileFromWorkerResourceSpec.

@Test
public void testGenerateDefaultSlotProfileFromWorkerResourceSpec() {
    final int numSlots = 5;
    final ResourceProfile resourceProfile = ResourceProfile.newBuilder().setCpuCores(1.0).setTaskHeapMemoryMB(1).setTaskOffHeapMemoryMB(2).setNetworkMemoryMB(3).setManagedMemoryMB(4).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1)).build();
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(1.0 * numSlots).setTaskHeapMemoryMB(1 * numSlots).setTaskOffHeapMemoryMB(2 * numSlots).setNetworkMemoryMB(3 * numSlots).setManagedMemoryMB(4 * numSlots).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, numSlots)).build();
    assertThat(SlotManagerUtils.generateDefaultSlotResourceProfile(workerResourceSpec, numSlots), is(resourceProfile));
}