use of com.twitter.heron.spi.packing.InstanceId in project heron by twitter.
the class PackingPlanBuilder method buildContainerPlans.
/**
* Estimate the per instance and topology resources for the packing plan based on the ramMap,
* instance defaults and paddingPercentage.
*
* @return container plans
*/
private static Set<PackingPlan.ContainerPlan> buildContainerPlans(Map<Integer, Container> containerInstances, Map<String, ByteAmount> ramMap, Resource instanceDefaults, int paddingPercentage) {
Set<PackingPlan.ContainerPlan> containerPlans = new LinkedHashSet<>();
for (Integer containerId : containerInstances.keySet()) {
Container container = containerInstances.get(containerId);
if (container.getInstances().size() == 0) {
continue;
}
ByteAmount containerRam = ByteAmount.ZERO;
ByteAmount containerDiskInBytes = ByteAmount.ZERO;
double containerCpu = 0;
// Calculate the resource required for single instance
Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
for (PackingPlan.InstancePlan instancePlan : container.getInstances()) {
InstanceId instanceId = new InstanceId(instancePlan.getComponentName(), instancePlan.getTaskId(), instancePlan.getComponentIndex());
ByteAmount instanceRam;
if (ramMap.containsKey(instanceId.getComponentName())) {
instanceRam = ramMap.get(instanceId.getComponentName());
} else {
instanceRam = instanceDefaults.getRam();
}
containerRam = containerRam.plus(instanceRam);
// Currently not yet support disk or cpu config for different components,
// so just use the default value.
ByteAmount instanceDisk = instanceDefaults.getDisk();
containerDiskInBytes = containerDiskInBytes.plus(instanceDisk);
double instanceCpu = instanceDefaults.getCpu();
containerCpu += instanceCpu;
// Insert it into the map
instancePlans.add(new PackingPlan.InstancePlan(instanceId, new Resource(instanceCpu, instanceRam, instanceDisk)));
}
containerCpu += (paddingPercentage * containerCpu) / 100;
containerRam = containerRam.increaseBy(paddingPercentage);
containerDiskInBytes = containerDiskInBytes.increaseBy(paddingPercentage);
Resource resource = new Resource(Math.round(containerCpu), containerRam, containerDiskInBytes);
PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
containerPlans.add(containerPlan);
}
return containerPlans;
}
use of com.twitter.heron.spi.packing.InstanceId in project heron by twitter.
the class RoundRobinPacking method getInstancesRamMapInContainer.
/**
* Calculate the ram required by any instance in the container
*
* @param allocation the allocation of instances in different container
* @return A map: (containerId -> (instanceId -> instanceRequiredRam))
*/
private Map<Integer, Map<InstanceId, ByteAmount>> getInstancesRamMapInContainer(Map<Integer, List<InstanceId>> allocation) {
Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMapInContainer = new HashMap<>();
for (int containerId : allocation.keySet()) {
List<InstanceId> instanceIds = allocation.get(containerId);
Map<InstanceId, ByteAmount> ramInsideContainer = new HashMap<>();
instancesRamMapInContainer.put(containerId, ramInsideContainer);
List<InstanceId> instancesToBeAccounted = new ArrayList<>();
// Calculate the actual value
ByteAmount usedRam = ByteAmount.ZERO;
for (InstanceId instanceId : instanceIds) {
String componentName = instanceId.getComponentName();
if (ramMap.containsKey(componentName)) {
ByteAmount ram = ramMap.get(componentName);
ramInsideContainer.put(instanceId, ram);
usedRam = usedRam.plus(ram);
} else {
instancesToBeAccounted.add(instanceId);
}
}
// Now we have calculated ram for instances specified in ComponentRamMap
// Then to calculate ram for the rest instances
ByteAmount containerRamHint = getContainerRamHint(allocation);
int instancesToAllocate = instancesToBeAccounted.size();
if (instancesToAllocate != 0) {
ByteAmount individualInstanceRam = instanceRamDefault;
// If container ram is specified
if (!containerRamHint.equals(NOT_SPECIFIED_NUMBER_VALUE)) {
// remove ram for heron internal process
ByteAmount remainingRam = containerRamHint.minus(DEFAULT_RAM_PADDING_PER_CONTAINER).minus(usedRam);
// Split remaining ram evenly
individualInstanceRam = remainingRam.divide(instancesToAllocate);
}
// Put the results in instancesRam
for (InstanceId instanceId : instancesToBeAccounted) {
ramInsideContainer.put(instanceId, individualInstanceRam);
}
}
}
return instancesRamMapInContainer;
}
use of com.twitter.heron.spi.packing.InstanceId in project heron by twitter.
the class RoundRobinPacking method getRoundRobinAllocation.
/**
* Get the instances' allocation basing on round robin algorithm
*
* @return containerId -> list of InstanceId belonging to this container
*/
private Map<Integer, List<InstanceId>> getRoundRobinAllocation() {
Map<Integer, List<InstanceId>> allocation = new HashMap<>();
int numContainer = TopologyUtils.getNumContainers(topology);
int totalInstance = TopologyUtils.getTotalInstance(topology);
if (numContainer > totalInstance) {
throw new RuntimeException("More containers allocated than instance.");
}
for (int i = 1; i <= numContainer; ++i) {
allocation.put(i, new ArrayList<InstanceId>());
}
int index = 1;
int globalTaskIndex = 1;
Map<String, Integer> parallelismMap = TopologyUtils.getComponentParallelism(topology);
for (String component : parallelismMap.keySet()) {
int numInstance = parallelismMap.get(component);
for (int i = 0; i < numInstance; ++i) {
allocation.get(index).add(new InstanceId(component, globalTaskIndex, i));
index = (index == numContainer) ? 1 : index + 1;
globalTaskIndex++;
}
}
return allocation;
}
use of com.twitter.heron.spi.packing.InstanceId in project heron by twitter.
the class CommonPackingTests method testScaleDownHomogenousFirst.
@Test
public void testScaleDownHomogenousFirst() throws Exception {
@SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] initialComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)), new Pair<>(3, new InstanceId(B, 4, 1)), new Pair<>(3, new InstanceId(B, 5, 2)), new Pair<>(3, new InstanceId(B, 6, 3)), new Pair<>(3, new InstanceId(B, 7, 4)) };
Map<String, Integer> componentChanges = new HashMap<>();
componentChanges.put(B, -4);
@SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] expectedComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)) };
doScaleDownTest(initialComponentInstances, componentChanges, expectedComponentInstances);
}
use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.
the class RoundRobinPacking method getRoundRobinAllocation.
/**
* Get the instances' allocation basing on round robin algorithm
*
* @return containerId -> list of InstanceId belonging to this container
*/
private Map<Integer, List<InstanceId>> getRoundRobinAllocation(int numContainer, Map<String, Integer> parallelismMap) {
Map<Integer, List<InstanceId>> allocation = new HashMap<>();
int totalInstance = TopologyUtils.getTotalInstance(parallelismMap);
if (numContainer > totalInstance) {
throw new RuntimeException("More containers allocated than instance.");
}
for (int i = 1; i <= numContainer; ++i) {
allocation.put(i, new ArrayList<InstanceId>());
}
int index = 1;
int globalTaskIndex = 1;
for (String component : parallelismMap.keySet()) {
int numInstance = parallelismMap.get(component);
for (int i = 0; i < numInstance; ++i) {
allocation.get(index).add(new InstanceId(component, globalTaskIndex, i));
index = (index == numContainer) ? 1 : index + 1;
globalTaskIndex++;
}
}
return allocation;
}
Aggregations