Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 26 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingTestHelper method generateTestPackingPlan.

/**
 * Returns a PackingPlan to use for testing scale up/down that has instances of specific
 * components on specific containers.
 */
private static PackingPlan generateTestPackingPlan(String topologyName, PackingPlan previousPackingPlan, Pair<Integer, String>[] addInstances, Pair<Integer, String>[] removeInstances, int containerPadding) throws ConstraintViolationException {
    PackingPlanBuilder builder = new PackingPlanBuilder(topologyName, previousPackingPlan);
    int instanceCount = 0;
    if (previousPackingPlan != null) {
        instanceCount = previousPackingPlan.getInstanceCount();
    } else if (addInstances != null) {
        instanceCount = addInstances.length;
    }
    // use basic default resource to allow all instances to fit on a single container, if that's
    // what the tester desired. We can extend this to permit passing custom resource requirements
    // as needed.
    Resource defaultInstanceResource = new Resource(1, ByteAmount.fromMegabytes(192), ByteAmount.fromMegabytes(1));
    Map<String, Resource> componentResourceMap = PackingUtils.getComponentResourceMap(toParallelismMap(previousPackingPlan, addInstances, removeInstances).keySet(), new HashMap<>(), new HashMap<>(), new HashMap<>(), defaultInstanceResource);
    builder.setDefaultInstanceResource(defaultInstanceResource);
    builder.setRequestedComponentResource(componentResourceMap);
    builder.setMaxContainerResource(new Resource(instanceCount, ByteAmount.fromMegabytes(192).multiply(instanceCount), ByteAmount.fromMegabytes(instanceCount)));
    builder.setInstanceConstraints(Collections.singletonList(new MinRamConstraint()));
    builder.setPackingConstraints(Collections.singletonList(new ResourceConstraint()));
    if (addInstances != null) {
        for (Pair<Integer, String> componentInstance : addInstances) {
            builder.addInstance(componentInstance.first, componentInstance.second);
        }
    }
    if (removeInstances != null) {
        for (Pair<Integer, String> componentInstance : removeInstances) {
            builder.removeInstance(componentInstance.first, componentInstance.second);
        }
    }
    return builder.build();
}

Example 27 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class AbstractPacking method setPackingConfigs.

/**
 * Instatiate the packing algorithm parameters related to this topology.
 */
private void setPackingConfigs(Config config) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    // instance default resources are acquired from heron system level config
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    int paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    ByteAmount ramPadding = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING);
    double cpuPadding = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_CPU_PADDING);
    Resource preliminaryPadding = new Resource(cpuPadding, ramPadding, PackingUtils.DEFAULT_CONTAINER_DISK_PADDING);
    this.maxNumInstancesPerContainer = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_NUM_INSTANCES, PackingUtils.DEFAULT_MAX_NUM_INSTANCES_PER_CONTAINER);
    // container default resources are computed as:
    // max number of instances per container * default instance resources
    double containerDefaultCpu = this.defaultInstanceResources.getCpu() * maxNumInstancesPerContainer;
    ByteAmount containerDefaultRam = this.defaultInstanceResources.getRam().multiply(maxNumInstancesPerContainer);
    ByteAmount containerDefaultDisk = this.defaultInstanceResources.getDisk().multiply(maxNumInstancesPerContainer);
    double containerCpu = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_REQUESTED, containerDefaultCpu);
    ByteAmount containerRam = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_REQUESTED, containerDefaultRam);
    ByteAmount containerDisk = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_DISK_REQUESTED, containerDefaultDisk);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // finalize padding
    this.padding = PackingUtils.finalizePadding(containerResource, preliminaryPadding, paddingPercentage);
    // finalize container resources
    this.maxContainerResources = containerResource;
    this.componentResourceMap = PackingUtils.getComponentResourceMap(TopologyUtils.getComponentParallelism(topology).keySet(), TopologyUtils.getComponentRamMapConfig(topology), TopologyUtils.getComponentCpuMapConfig(topology), TopologyUtils.getComponentDiskMapConfig(topology), defaultInstanceResources);
}

Example 28 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testPartialRamMapWithoutContainerRequestedResources.

@Test
public void testPartialRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        int instancesCount = containerPlan.getInstances().size();
        if (instancesCount == 4) {
            // Biggest container
            Assert.assertEquals(1, differentResources.size());
        } else {
            // Smaller container
            Assert.assertEquals(2, differentResources.size());
        }
    }
}

Example 29 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testHugePartialRamMapWithoutContainerRequestedResources.

// Throw an error if default container resource (default instance resource * number of instances
// + padding) is not enough.
@Test(expected = PackingException.class)
public void testHugePartialRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    ByteAmount boltRam = ByteAmount.fromGigabytes(10);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, differentResources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), differentResources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, differentResources.iterator().next().getCpu(), DELTA);
    }
}

Example 30 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testNoRamMapWithoutContainerRequestedResources.

@Test
public void testNoRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Container RAM is not set in config
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.empty(), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, differentResources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), differentResources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, differentResources.iterator().next().getCpu(), DELTA);
    }
}