use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class Container method getTotalUsedResources.
/**
* Computes the used resources of the container by taking into account the resources
* allocated for each instance.
*
* @return a Resource object that describes the used cpu, ram and disk in the container.
*/
private Resource getTotalUsedResources() {
ByteAmount usedRam = ByteAmount.ZERO;
double usedCpuCores = 0;
ByteAmount usedDisk = ByteAmount.ZERO;
for (PackingPlan.InstancePlan instancePlan : this.instances) {
Resource resource = instancePlan.getResource();
usedRam = usedRam.plus(resource.getRam());
usedCpuCores += resource.getCpu();
usedDisk = usedDisk.plus(resource.getDisk());
}
return new Resource(usedCpuCores, usedRam, usedDisk);
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class FirstFitDecreasingPacking method setPackingConfigs.
/**
* Instatiate the packing algorithm parameters related to this topology.
*/
private void setPackingConfigs(Config config) {
List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
this.paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
this.maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_CPU_HINT, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_RAM_HINT, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_DISK_HINT, defaultDisk.increaseBy(paddingPercentage)));
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class FirstFitDecreasingPacking method getSortedRAMInstances.
/**
* Sort the components in decreasing order based on their RAM requirements
*
* @return The sorted list of components and their RAM requirements
*/
private ArrayList<RamRequirement> getSortedRAMInstances(Set<String> componentNames) {
ArrayList<RamRequirement> ramRequirements = new ArrayList<>();
Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
for (String componentName : componentNames) {
Resource requiredResource = PackingUtils.getResourceRequirement(componentName, ramMap, this.defaultInstanceResources, this.maxContainerResources, this.paddingPercentage);
ramRequirements.add(new RamRequirement(componentName, requiredResource.getRam()));
}
Collections.sort(ramRequirements, Collections.reverseOrder());
return ramRequirements;
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class FirstFitDecreasingPackingTest method testDefaultContainerSizeWithPadding.
/**
* Test the scenario where the max container size is the default but padding is configured
*/
@Test
public void testDefaultContainerSizeWithPadding() throws Exception {
int padding = 50;
int defaultNumInstancesperContainer = 4;
topologyConfig.setContainerPaddingPercentage(padding);
TopologyAPI.Topology newTopology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlan = pack(newTopology);
Assert.assertEquals(2, packingPlan.getContainers().size());
Assert.assertEquals(totalInstances, packingPlan.getInstanceCount());
ByteAmount defaultRam = instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer).increaseBy(padding);
AssertPacking.assertContainerRam(packingPlan.getContainers(), defaultRam);
AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, 3);
AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, 4);
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class FirstFitDecreasingPackingTest method scaleDownAndUpNoPadding.
/**
* Test the scenario where scaling down and up is simultaneously requested and padding is
* configured
*/
@Test
public void scaleDownAndUpNoPadding() throws Exception {
int paddingPercentage = 0;
topologyConfig.setContainerPaddingPercentage(paddingPercentage);
ByteAmount spoutRam = ByteAmount.fromGigabytes(4);
ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
topologyConfig.setContainerMaxRamHint(maxContainerRam);
topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
int noBolts = 3;
int noSpouts = 1;
TopologyAPI.Topology topologyExplicitRamMap = getTopology(noSpouts, noBolts, topologyConfig);
int spoutScalingUp = 1;
int boltScalingDown = -1;
Map<String, Integer> componentChanges = new HashMap<>();
// 2 spouts
componentChanges.put(SPOUT_NAME, spoutScalingUp);
// 2 bolts
componentChanges.put(BOLT_NAME, boltScalingDown);
int numContainersBeforeRepack = 1;
PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, instanceDefaultResources.getRam(), noBolts, spoutRam, noSpouts, numContainersBeforeRepack, noSpouts + noBolts);
Assert.assertEquals(2, newPackingPlan.getContainers().size());
Assert.assertEquals((Integer) (noSpouts + noBolts + spoutScalingUp + boltScalingDown), newPackingPlan.getInstanceCount());
AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, noBolts + boltScalingDown);
AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, noSpouts + spoutScalingUp);
}
Aggregations