Examples with CPUResource org.apache.flink.api.common.resources.CPUResource used on opensource projects

Example 6 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class TaskExecutorProcessSpecTest method testNotEquals.

@Test
public void testNotEquals() {
    TaskExecutorProcessSpec spec1 = new TaskExecutorProcessSpec(new CPUResource(1.0), MemorySize.parse("1m"), MemorySize.parse("2m"), MemorySize.parse("3m"), MemorySize.parse("4m"), MemorySize.parse("5m"), MemorySize.parse("6m"), MemorySize.parse("7m"), MemorySize.parse("8m"), Collections.singleton(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1)));
    TaskExecutorProcessSpec spec2 = new TaskExecutorProcessSpec(new CPUResource(0.0), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    assertThat(spec1, not(spec2));
}

Example 7 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class YarnResourceManagerDriverTest method testStartWorkerVariousSpec.

@Test
public void testStartWorkerVariousSpec() throws Exception {
    final TaskExecutorProcessSpec taskExecutorProcessSpec1 = new TaskExecutorProcessSpec(new CPUResource(1), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    final TaskExecutorProcessSpec taskExecutorProcessSpec2 = new TaskExecutorProcessSpec(new CPUResource(2), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    new Context() {

        {
            final String startCommand1 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (50L << 20);
            final String startCommand2 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (100L << 20);
            final CompletableFuture<Void> startContainerAsyncCommandFuture1 = new CompletableFuture<>();
            final CompletableFuture<Void> startContainerAsyncCommandFuture2 = new CompletableFuture<>();
            prepareForTestStartTaskExecutorProcessVariousSpec(startCommand1, startCommand2, startContainerAsyncCommandFuture1, startContainerAsyncCommandFuture2, taskExecutorProcessSpec1);
            testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(tuple -> {
                final Priority priority = tuple.f0;
                final List<AMRMClient.ContainerRequest> matchingRequests = new ArrayList<>();
                for (CompletableFuture<AMRMClient.ContainerRequest> addContainerRequestFuture : addContainerRequestFutures) {
                    final AMRMClient.ContainerRequest request = addContainerRequestFuture.getNow(null);
                    if (request != null && priority.equals(request.getPriority())) {
                        assertThat(tuple.f2, is(request.getCapability()));
                        matchingRequests.add(request);
                    }
                }
                return Collections.singletonList(matchingRequests);
            });
            runTest(() -> {
                final Resource containerResource1 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get();
                final Resource containerResource2 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get();
                // Make sure two worker resource spec will be normalized to different
                // container resources
                assertNotEquals(containerResource1, containerResource2);
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec1));
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec2));
                // Verify both containers requested
                verifyFutureCompleted(addContainerRequestFutures.get(0));
                verifyFutureCompleted(addContainerRequestFutures.get(1));
                // Mock that container 1 is allocated
                Container container1 = createTestingContainerWithResource(containerResource1);
                resourceManagerClientCallbackHandler.onContainersAllocated(Collections.singletonList(container1));
                // Verify that only worker with spec1 is started.
                verifyFutureCompleted(startContainerAsyncCommandFuture1);
                assertFalse(startContainerAsyncCommandFuture2.isDone());
                // Mock that container 1 is completed, while the worker is still pending
                ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(container1.getId());
                resourceManagerClientCallbackHandler.onContainersCompleted(Collections.singletonList(testingContainerStatus));
                // Verify that only container 1 is requested again
                verifyFutureCompleted(addContainerRequestFutures.get(2));
                assertThat(addContainerRequestFutures.get(2).get().getCapability(), is(containerResource1));
                assertFalse(addContainerRequestFutures.get(3).isDone());
            });
        }
    };
}

Example 8 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class YarnWorkerResourceSpecFactoryTest method testGetCpuCoresYarnOption.

@Test
public void testGetCpuCoresYarnOption() {
    final Configuration configuration = new Configuration();
    configuration.setInteger(YarnConfigOptions.VCORES, 2);
    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 3);
    assertThat(YarnWorkerResourceSpecFactory.getDefaultCpus(configuration), is(new CPUResource(2.0)));
}

Example 9 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class YarnWorkerResourceSpecFactoryTest method testGetCpuCoresCommonOption.

@Test
public void testGetCpuCoresCommonOption() {
    final Configuration configuration = new Configuration();
    configuration.setDouble(TaskManagerOptions.CPU_CORES, 1.0);
    configuration.setInteger(YarnConfigOptions.VCORES, 2);
    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 3);
    assertThat(YarnWorkerResourceSpecFactory.getDefaultCpus(configuration), is(new CPUResource(1.0)));
}

Example 10 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class SlotManagerUtilsTest method testGenerateDefaultSlotConsistentWithTaskExecutorResourceUtils.

@Test
public void testGenerateDefaultSlotConsistentWithTaskExecutorResourceUtils() {
    final int numSlots = 5;
    final TaskExecutorResourceSpec taskExecutorResourceSpec = new TaskExecutorResourceSpec(new CPUResource(1.0), MemorySize.parse("1m"), MemorySize.parse("2m"), MemorySize.parse("3m"), MemorySize.parse("4m"), Collections.singleton(new ExternalResource(EXTERNAL_RESOURCE_NAME, numSlots)));
    final ResourceProfile resourceProfileFromTaskExecutorResourceUtils = TaskExecutorResourceUtils.generateDefaultSlotResourceProfile(taskExecutorResourceSpec, numSlots);
    final ResourceProfile totalResourceProfile = TaskExecutorResourceUtils.generateTotalAvailableResourceProfile(taskExecutorResourceSpec);
    final WorkerResourceSpec workerResourceSpec = WorkerResourceSpec.fromTotalResourceProfile(totalResourceProfile, numSlots);
    assertThat(SlotManagerUtils.generateDefaultSlotResourceProfile(totalResourceProfile, numSlots), is(resourceProfileFromTaskExecutorResourceUtils));
    assertThat(SlotManagerUtils.generateDefaultSlotResourceProfile(workerResourceSpec, numSlots), is(resourceProfileFromTaskExecutorResourceUtils));
}