Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 1 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtils method getComponentResourceMap.

/**
 * Compose the component resource map by reading from user configs or default
 *
 * @param components component names
 * @param componentRamMap user configured component ram map
 * @param componentCpuMap user configured component cpu map
 * @param componentDiskMap user configured component disk map
 * @param defaultInstanceResource default instance resources
 * @return component resource map
 */
public static Map<String, Resource> getComponentResourceMap(Set<String> components, Map<String, ByteAmount> componentRamMap, Map<String, Double> componentCpuMap, Map<String, ByteAmount> componentDiskMap, Resource defaultInstanceResource) {
    Map<String, Resource> componentResourceMap = new HashMap<>();
    for (String component : components) {
        ByteAmount instanceRam = componentRamMap.getOrDefault(component, defaultInstanceResource.getRam());
        double instanceCpu = componentCpuMap.getOrDefault(component, defaultInstanceResource.getCpu());
        ByteAmount instanceDisk = componentDiskMap.getOrDefault(component, defaultInstanceResource.getDisk());
        componentResourceMap.put(component, new Resource(instanceCpu, instanceRam, instanceDisk));
    }
    return componentResourceMap;
}

Example 2 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class CommonPackingTests method setUp.

@Before
public void setUp() {
    this.spoutParallelism = 4;
    this.boltParallelism = 3;
    this.totalInstances = this.spoutParallelism + this.boltParallelism;
    // Set up the topology and its config. Tests can safely modify the config by reference after the
    // topology is created, but those changes will not be reflected in the underlying protobuf
    // object Config and Topology objects. This is typically fine for packing tests since they don't
    // access the protobuf values.
    this.topologyConfig = new org.apache.heron.api.Config();
    this.topologyConfig.setTopologyContainerMaxNumInstances(4);
    this.topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    Config config = PackingTestUtils.newTestConfig(this.topology);
    this.instanceDefaultResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
}

Example 3 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testContainerRequestedResourcesWhenRamPaddingSet.

/**
 * Test the scenario container level resource config are set
 */
@Test
public void testContainerRequestedResourcesWhenRamPaddingSet() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount containerRamPadding = ByteAmount.fromMegabytes(512);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setContainerRamPadding(containerRamPadding);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.empty(), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding().cloneWithRam(containerRamPadding), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(containerRamPadding).divide(instancesCount), resources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, resources.iterator().next().getCpu(), DELTA);
    }
}

Example 4 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testFullRamMapWithoutContainerRequestedResources.

@Test
public void testFullRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount spoutRam = ByteAmount.fromMegabytes(500);
    ByteAmount boltRam = ByteAmount.fromMegabytes(1000);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.of(spoutRam), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        // Bolt and spout ram sizes are both fixed.
        Assert.assertEquals(2, differentResources.size());
    }
}

Example 5 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested.

/**
 * Test the scenario RAM map config is fully set
 */
@Test
public void testCompleteRamMapRequested() throws Exception {
    // Explicit set max resources for container
    // the value should be ignored, since we set the complete component RAM map
    ByteAmount containerRam = ByteAmount.fromGigabytes(15);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources.cloneWithRam(spoutRam), spoutParallelism, 2, containerResource);
}