Search in sources :

Example 31 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class Container method getTotalUsedResources.

/**
 * Computes the used resources of the container by taking into account the resources
 * allocated for each instance.
 *
 * @return a Resource object that describes the used cpu, ram and disk in the container.
 */
private Resource getTotalUsedResources() {
    ByteAmount usedRam = ByteAmount.ZERO;
    double usedCpuCores = 0;
    ByteAmount usedDisk = ByteAmount.ZERO;
    for (PackingPlan.InstancePlan instancePlan : this.instances) {
        Resource resource = instancePlan.getResource();
        usedRam = usedRam.plus(resource.getRam());
        usedCpuCores += resource.getCpu();
        usedDisk = usedDisk.plus(resource.getDisk());
    }
    return new Resource(usedCpuCores, usedRam, usedDisk);
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) Resource(com.twitter.heron.spi.packing.Resource)

Example 32 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class FirstFitDecreasingPacking method setPackingConfigs.

/**
 * Instatiate the packing algorithm parameters related to this topology.
 */
private void setPackingConfigs(Config config) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    this.paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    this.maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_CPU_HINT, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_RAM_HINT, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_DISK_HINT, defaultDisk.increaseBy(paddingPercentage)));
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) Resource(com.twitter.heron.spi.packing.Resource) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI)

Example 33 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class FirstFitDecreasingPacking method getSortedRAMInstances.

/**
 * Sort the components in decreasing order based on their RAM requirements
 *
 * @return The sorted list of components and their RAM requirements
 */
private ArrayList<RamRequirement> getSortedRAMInstances(Set<String> componentNames) {
    ArrayList<RamRequirement> ramRequirements = new ArrayList<>();
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    for (String componentName : componentNames) {
        Resource requiredResource = PackingUtils.getResourceRequirement(componentName, ramMap, this.defaultInstanceResources, this.maxContainerResources, this.paddingPercentage);
        ramRequirements.add(new RamRequirement(componentName, requiredResource.getRam()));
    }
    Collections.sort(ramRequirements, Collections.reverseOrder());
    return ramRequirements;
}
Also used : RamRequirement(com.twitter.heron.packing.RamRequirement) ByteAmount(com.twitter.heron.common.basics.ByteAmount) ArrayList(java.util.ArrayList) Resource(com.twitter.heron.spi.packing.Resource)

Example 34 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class FirstFitDecreasingPackingTest method testDefaultContainerSizeWithPadding.

/**
 * Test the scenario where the max container size is the default but padding is configured
 */
@Test
public void testDefaultContainerSizeWithPadding() throws Exception {
    int padding = 50;
    int defaultNumInstancesperContainer = 4;
    topologyConfig.setContainerPaddingPercentage(padding);
    TopologyAPI.Topology newTopology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = pack(newTopology);
    Assert.assertEquals(2, packingPlan.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlan.getInstanceCount());
    ByteAmount defaultRam = instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer).increaseBy(padding);
    AssertPacking.assertContainerRam(packingPlan.getContainers(), defaultRam);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, 4);
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) Test(org.junit.Test)

Example 35 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class FirstFitDecreasingPackingTest method scaleDownAndUpNoPadding.

/**
 * Test the scenario where scaling down and up is simultaneously requested and padding is
 * configured
 */
@Test
public void scaleDownAndUpNoPadding() throws Exception {
    int paddingPercentage = 0;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(4);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    int noBolts = 3;
    int noSpouts = 1;
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(noSpouts, noBolts, topologyConfig);
    int spoutScalingUp = 1;
    int boltScalingDown = -1;
    Map<String, Integer> componentChanges = new HashMap<>();
    // 2 spouts
    componentChanges.put(SPOUT_NAME, spoutScalingUp);
    // 2 bolts
    componentChanges.put(BOLT_NAME, boltScalingDown);
    int numContainersBeforeRepack = 1;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, instanceDefaultResources.getRam(), noBolts, spoutRam, noSpouts, numContainersBeforeRepack, noSpouts + noBolts);
    Assert.assertEquals(2, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (noSpouts + noBolts + spoutScalingUp + boltScalingDown), newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, noBolts + boltScalingDown);
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, noSpouts + spoutScalingUp);
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) HashMap(java.util.HashMap) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) Test(org.junit.Test)

Aggregations

ByteAmount (com.twitter.heron.common.basics.ByteAmount)109 Test (org.junit.Test)64 PackingPlan (com.twitter.heron.spi.packing.PackingPlan)61 TopologyAPI (com.twitter.heron.api.generated.TopologyAPI)55 HashMap (java.util.HashMap)30 Resource (com.twitter.heron.spi.packing.Resource)29 HashSet (java.util.HashSet)15 Config (com.twitter.heron.api.Config)9 Config (com.twitter.heron.spi.common.Config)9 TreeMap (java.util.TreeMap)9 InstanceId (com.twitter.heron.spi.packing.InstanceId)6 ArrayList (java.util.ArrayList)6 VisibleForTesting (com.google.common.annotations.VisibleForTesting)4 Map (java.util.Map)4 EvaluatorRequest (org.apache.reef.driver.evaluator.EvaluatorRequest)4 List (java.util.List)3 RamRequirement (com.twitter.heron.packing.RamRequirement)2 ResourceExceededException (com.twitter.heron.packing.ResourceExceededException)2 PackingPlanBuilder (com.twitter.heron.packing.builder.PackingPlanBuilder)2 BaseContainer (com.twitter.heron.scheduler.mesos.framework.BaseContainer)2