Examples with ExternalResource org.apache.flink.api.common.resources.ExternalResource used on opensource projects

Example 6 with ExternalResource

use of org.apache.flink.api.common.resources.ExternalResource in project flink by apache.

the class SlotManagerUtilsTest method testGenerateDefaultSlotConsistentWithTaskExecutorResourceUtils.

@Test
public void testGenerateDefaultSlotConsistentWithTaskExecutorResourceUtils() {
    final int numSlots = 5;
    final TaskExecutorResourceSpec taskExecutorResourceSpec = new TaskExecutorResourceSpec(new CPUResource(1.0), MemorySize.parse("1m"), MemorySize.parse("2m"), MemorySize.parse("3m"), MemorySize.parse("4m"), Collections.singleton(new ExternalResource(EXTERNAL_RESOURCE_NAME, numSlots)));
    final ResourceProfile resourceProfileFromTaskExecutorResourceUtils = TaskExecutorResourceUtils.generateDefaultSlotResourceProfile(taskExecutorResourceSpec, numSlots);
    final ResourceProfile totalResourceProfile = TaskExecutorResourceUtils.generateTotalAvailableResourceProfile(taskExecutorResourceSpec);
    final WorkerResourceSpec workerResourceSpec = WorkerResourceSpec.fromTotalResourceProfile(totalResourceProfile, numSlots);
    assertThat(SlotManagerUtils.generateDefaultSlotResourceProfile(totalResourceProfile, numSlots), is(resourceProfileFromTaskExecutorResourceUtils));
    assertThat(SlotManagerUtils.generateDefaultSlotResourceProfile(workerResourceSpec, numSlots), is(resourceProfileFromTaskExecutorResourceUtils));
}

Example 7 with ExternalResource

use of org.apache.flink.api.common.resources.ExternalResource in project flink by apache.

the class ResourceSpecTest method testSubtract.

@Test
public void testSubtract() {
    final ResourceSpec rs1 = ResourceSpec.newBuilder(1.0, 100).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1.1)).build();
    final ResourceSpec rs2 = ResourceSpec.newBuilder(0.2, 100).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, 0.5)).build();
    final ResourceSpec subtracted = rs1.subtract(rs2);
    assertEquals(new CPUResource(0.8), subtracted.getCpuCores());
    assertEquals(0, subtracted.getTaskHeapMemory().getMebiBytes());
    assertEquals(new ExternalResource(EXTERNAL_RESOURCE_NAME, 0.6), subtracted.getExtendedResource(EXTERNAL_RESOURCE_NAME).get());
}

Example 8 with ExternalResource

use of org.apache.flink.api.common.resources.ExternalResource in project flink by apache.

the class ResourceSpecTest method testMerge.

@Test
public void testMerge() throws Exception {
    ResourceSpec rs1 = ResourceSpec.newBuilder(1.0, 100).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1.1)).build();
    ResourceSpec rs2 = ResourceSpec.newBuilder(1.0, 100).build();
    ResourceSpec rs3 = rs1.merge(rs2);
    assertEquals(new CPUResource(2.0), rs3.getCpuCores());
    assertEquals(200, rs3.getTaskHeapMemory().getMebiBytes());
    assertEquals(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1.1), rs3.getExtendedResource(EXTERNAL_RESOURCE_NAME).get());
    ResourceSpec rs4 = rs1.merge(rs3);
    assertEquals(new ExternalResource(EXTERNAL_RESOURCE_NAME, 2.2), rs4.getExtendedResource(EXTERNAL_RESOURCE_NAME).get());
}

Example 9 with ExternalResource

use of org.apache.flink.api.common.resources.ExternalResource in project flink by apache.

the class KubernetesUtils method getResourceRequirements.

/**
 * Get resource requirements from memory and cpu.
 *
 * @param resourceRequirements resource requirements in pod template
 * @param mem Memory in mb.
 * @param memoryLimitFactor limit factor for the memory, used to set the limit resources.
 * @param cpu cpu.
 * @param cpuLimitFactor limit factor for the cpu, used to set the limit resources.
 * @param externalResources external resources
 * @param externalResourceConfigKeys config keys of external resources
 * @return KubernetesResource requirements.
 */
public static ResourceRequirements getResourceRequirements(ResourceRequirements resourceRequirements, int mem, double memoryLimitFactor, double cpu, double cpuLimitFactor, Map<String, ExternalResource> externalResources, Map<String, String> externalResourceConfigKeys) {
    final Quantity cpuQuantity = new Quantity(String.valueOf(cpu));
    final Quantity cpuLimitQuantity = new Quantity(String.valueOf(cpu * cpuLimitFactor));
    final Quantity memQuantity = new Quantity(mem + Constants.RESOURCE_UNIT_MB);
    final Quantity memQuantityLimit = new Quantity(((int) (mem * memoryLimitFactor)) + Constants.RESOURCE_UNIT_MB);
    ResourceRequirementsBuilder resourceRequirementsBuilder = new ResourceRequirementsBuilder(resourceRequirements).addToRequests(Constants.RESOURCE_NAME_MEMORY, memQuantity).addToRequests(Constants.RESOURCE_NAME_CPU, cpuQuantity).addToLimits(Constants.RESOURCE_NAME_MEMORY, memQuantityLimit).addToLimits(Constants.RESOURCE_NAME_CPU, cpuLimitQuantity);
    // Add the external resources to resource requirement.
    for (Map.Entry<String, ExternalResource> externalResource : externalResources.entrySet()) {
        final String configKey = externalResourceConfigKeys.get(externalResource.getKey());
        if (!StringUtils.isNullOrWhitespaceOnly(configKey)) {
            final Quantity resourceQuantity = new Quantity(String.valueOf(externalResource.getValue().getValue().longValue()));
            resourceRequirementsBuilder.addToRequests(configKey, resourceQuantity).addToLimits(configKey, resourceQuantity);
            LOG.info("Request external resource {} with config key {}.", resourceQuantity.getAmount(), configKey);
        }
    }
    return resourceRequirementsBuilder.build();
}

Example 10 with ExternalResource

use of org.apache.flink.api.common.resources.ExternalResource in project flink by apache.

the class SlotSharingGroupUtilsTest method testCovertToResourceSpec.

@Test
public void testCovertToResourceSpec() {
    final ExternalResource gpu = new ExternalResource("gpu", 1);
    final ResourceSpec resourceSpec = ResourceSpec.newBuilder(1.0, 100).setManagedMemoryMB(200).setTaskOffHeapMemoryMB(300).setExtendedResource(gpu).build();
    final SlotSharingGroup slotSharingGroup1 = SlotSharingGroup.newBuilder("ssg").setCpuCores(resourceSpec.getCpuCores().getValue().doubleValue()).setTaskHeapMemory(resourceSpec.getTaskHeapMemory()).setTaskOffHeapMemory(resourceSpec.getTaskOffHeapMemory()).setManagedMemory(resourceSpec.getManagedMemory()).setExternalResource(gpu.getName(), gpu.getValue().doubleValue()).build();
    final SlotSharingGroup slotSharingGroup2 = SlotSharingGroup.newBuilder("ssg").build();
    assertThat(SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup1), is(resourceSpec));
    assertThat(SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup2), is(ResourceSpec.UNKNOWN));
}