use of org.apache.heron.spi.packing.InstanceId in project heron by twitter.
the class PackingPlanBuilder method addInstance.
// adds an instance to a container with id containerId. If that container does not exist, it will
// be lazily initialized, which could result in more containers than those requested using the
// updateNumContainers method
public PackingPlanBuilder addInstance(Integer containerId, String componentName) throws ConstraintViolationException {
// create container if not existed
initContainer(containerId);
Integer taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
Integer componentIndex = componentIndexes.containsKey(componentName) ? componentIndexes.get(componentName).last() + 1 : 0;
InstanceId instanceId = new InstanceId(componentName, taskId, componentIndex);
Resource instanceResource = componentResourceMap.getOrDefault(componentName, defaultInstanceResource);
Container container = containers.get(containerId);
PackingPlan.InstancePlan instancePlan = new PackingPlan.InstancePlan(instanceId, instanceResource);
// Check constraints
for (InstanceConstraint constraint : instanceConstraints) {
constraint.validate(instancePlan);
}
for (PackingConstraint constraint : packingConstraints) {
constraint.validate(container, instancePlan);
}
addToContainer(container, instancePlan, this.componentIndexes, this.taskIds);
LOG.finest(String.format("Added to container %d instance %s", containerId, instanceId));
return this;
}
use of org.apache.heron.spi.packing.InstanceId in project heron by twitter.
the class RoundRobinPacking method packInternal.
private PackingPlan packInternal(int numContainer, Map<String, Integer> parallelismMap) {
// Get the instances' round-robin allocation
Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation(numContainer, parallelismMap);
Resource containerResourceHint = getContainerResourceHint(roundRobinAllocation);
int largestContainerSize = getLargestContainerSize(roundRobinAllocation);
// Get the RAM map for every instance
ByteAmount containerRamDefault = instanceRamDefault.multiply(largestContainerSize).plus(containerRamPadding);
Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = calculateInstancesResourceMapInContainer(roundRobinAllocation, TopologyUtils.getComponentRamMapConfig(topology), containerResourceHint.getRam(), containerRamDefault, instanceRamDefault, containerRamPadding, ByteAmount.ZERO, NOT_SPECIFIED_BYTE_AMOUNT, RAM);
// Get the CPU map for every instance
float containerCPUDefault = Math.round(instanceCpuDefault * largestContainerSize + containerCpuPadding);
Map<Integer, Map<InstanceId, CPUShare>> instancesCpuMap = calculateInstancesResourceMapInContainer(roundRobinAllocation, CPUShare.convertDoubleMapToCpuShareMap(TopologyUtils.getComponentCpuMapConfig(topology)), CPUShare.fromDouble(containerResourceHint.getCpu()), CPUShare.fromDouble(containerCPUDefault), CPUShare.fromDouble(instanceCpuDefault), CPUShare.fromDouble(containerCpuPadding), CPUShare.fromDouble(0.0), CPUShare.fromDouble(NOT_SPECIFIED_CPU_SHARE), CPU);
LOG.info(String.format("Pack internal: container CPU hint: %.3f, RAM hint: %s, disk hint: %s.", containerResourceHint.getCpu(), containerResourceHint.getRam().toString(), containerResourceHint.getDisk().toString()));
// Construct the PackingPlan
Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
for (int containerId : roundRobinAllocation.keySet()) {
List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
// Calculate the resource required for single instance
Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
ByteAmount containerRam = ByteAmount.ZERO;
double containerCpu = 0.0;
for (InstanceId instanceId : instanceList) {
ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
Double instanceCpu = instancesCpuMap.get(containerId).get(instanceId).getValue();
// Currently not yet support disk config for different components, just use the default.
ByteAmount instanceDisk = instanceDiskDefault;
Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
// Insert it into the map
instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
containerRam = containerRam.plus(instanceRam);
containerCpu += instanceCpu;
}
// finalize container resource
containerCpu += containerCpuPadding;
if (containerResourceHint.getCpu() != NOT_SPECIFIED_CPU_SHARE) {
containerCpu = Math.min(containerCpu, containerResourceHint.getCpu());
}
containerRam = containerRam.plus(containerRamPadding);
if (!containerResourceHint.getRam().equals(NOT_SPECIFIED_BYTE_AMOUNT)) {
containerRam = ByteAmount.fromBytes(Math.min(containerRam.asBytes(), containerResourceHint.getRam().asBytes()));
}
ByteAmount containerDisk = containerResourceHint.getDisk();
if (containerDisk.equals(NOT_SPECIFIED_BYTE_AMOUNT)) {
containerDisk = instanceDiskDefault.multiply(largestContainerSize).plus(DEFAULT_DISK_PADDING_PER_CONTAINER);
}
Resource resource = new Resource(Math.max(containerCpu, containerResourceHint.getCpu()), containerRam, containerDisk);
PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
containerPlans.add(containerPlan);
LOG.info(String.format("Pack internal finalized: container#%d CPU: %f, RAM: %s, disk: %s.", containerId, resource.getCpu(), resource.getRam().toString(), resource.getDisk().toString()));
}
PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
validatePackingPlan(plan);
return plan;
}
use of org.apache.heron.spi.packing.InstanceId in project heron by twitter.
the class RoundRobinPacking method getRoundRobinAllocation.
/**
* Get the instances' allocation basing on round robin algorithm
*
* @return containerId -> list of InstanceId belonging to this container
*/
private Map<Integer, List<InstanceId>> getRoundRobinAllocation(int numContainer, Map<String, Integer> parallelismMap) {
Map<Integer, List<InstanceId>> allocation = new HashMap<>();
int totalInstance = TopologyUtils.getTotalInstance(parallelismMap);
if (numContainer < 1) {
throw new RuntimeException(String.format("Invlaid number of container: %d", numContainer));
} else if (numContainer > totalInstance) {
throw new RuntimeException(String.format("More containers (%d) allocated than instances (%d).", numContainer, totalInstance));
}
for (int i = 1; i <= numContainer; ++i) {
allocation.put(i, new ArrayList<>());
}
int index = 1;
int globalTaskIndex = 1;
// To ensure we spread out the big instances first
// Only sorting by RAM here because only RAM can be explicitly capped by JVM processes
List<String> sortedInstances = getSortedRAMComponents(parallelismMap.keySet()).stream().map(ResourceRequirement::getComponentName).collect(Collectors.toList());
for (String component : sortedInstances) {
int numInstance = parallelismMap.get(component);
for (int i = 0; i < numInstance; ++i) {
allocation.get(index).add(new InstanceId(component, globalTaskIndex, i));
index = (index == numContainer) ? 1 : index + 1;
globalTaskIndex++;
}
}
return allocation;
}
use of org.apache.heron.spi.packing.InstanceId in project heron by twitter.
the class RoundRobinPacking method calculateInstancesResourceMapInContainer.
@SuppressWarnings("unchecked")
private <T extends ResourceMeasure> Map<Integer, Map<InstanceId, T>> calculateInstancesResourceMapInContainer(Map<Integer, List<InstanceId>> allocation, Map<String, T> resMap, T containerResHint, T defaultContainerRes, T instanceResDefault, T containerResPadding, T zero, T notSpecified, String resourceType) {
Map<Integer, Map<InstanceId, T>> instancesResMapInContainer = new HashMap<>();
for (int containerId : allocation.keySet()) {
List<InstanceId> instanceIds = allocation.get(containerId);
Map<InstanceId, T> resInsideContainer = new HashMap<>();
instancesResMapInContainer.put(containerId, resInsideContainer);
List<InstanceId> unspecifiedInstances = new ArrayList<>();
// Register the instance resource allocation and calculate the used resource so far
T usedRes = zero;
for (InstanceId instanceId : instanceIds) {
String componentName = instanceId.getComponentName();
if (resMap.containsKey(componentName)) {
T res = resMap.get(componentName);
resInsideContainer.put(instanceId, res);
usedRes = (T) usedRes.plus(res);
} else {
unspecifiedInstances.add(instanceId);
}
}
// Soft padding constraint validation: warn if padding amount cannot be accommodated
boolean paddingThrottling = false;
if (!containerResHint.equals(notSpecified) && usedRes.greaterThan(containerResHint.minus(containerResPadding))) {
// Validate instance resources specified so far don't violate container-level constraint
if (usedRes.greaterThan(containerResHint)) {
throw new PackingException(String.format("Invalid packing plan generated. " + "Total instance %s (%s) in container#%d have exceeded " + "the container-level constraint of %s.", resourceType, usedRes.toString(), containerId, containerResHint.toString()));
}
paddingThrottling = true;
LOG.warning(String.format("Container#%d (max %s: %s) is now hosting instances that " + "take up to %s %s. The container may not have enough resource to accommodate " + "internal processes which take up to %s %s.", containerId, resourceType, containerResHint.toString(), usedRes.toString(), resourceType, containerResPadding.toString(), resourceType));
}
// calculate resource for the remaining unspecified instances if any
T containerRes = containerResHint;
if (containerResHint.equals(notSpecified)) {
containerRes = defaultContainerRes;
}
if (!unspecifiedInstances.isEmpty()) {
T individualInstanceRes = instanceResDefault;
// discount resource for heron internal process (padding) and used (usedRes)
T remainingRes;
if (paddingThrottling) {
remainingRes = (T) containerRes.minus(usedRes);
} else {
remainingRes = (T) containerRes.minus(containerResPadding).minus(usedRes);
}
if (remainingRes.lessOrEqual(zero)) {
throw new PackingException(String.format("Invalid packing plan generated. " + "No enough %s to allocate for unspecified instances", resourceType));
}
// Split remaining resource evenly
individualInstanceRes = (T) remainingRes.divide(unspecifiedInstances.size());
// Put the results in resInsideContainer
for (InstanceId instanceId : unspecifiedInstances) {
resInsideContainer.put(instanceId, individualInstanceRes);
}
}
}
return instancesResMapInContainer;
}
use of org.apache.heron.spi.packing.InstanceId in project heron by twitter.
the class AssertPacking method assertPackingPlan.
public static void assertPackingPlan(String expectedTopologyName, Pair<Integer, InstanceId>[] expectedComponentInstances, PackingPlan plan) {
assertEquals(expectedTopologyName, plan.getId());
assertEquals("Unexpected number of instances: " + plan.getContainers(), expectedComponentInstances.length, plan.getInstanceCount().intValue());
// for every instance on a given container...
Set<Integer> expectedContainerIds = new HashSet<>();
for (Pair<Integer, InstanceId> expectedComponentInstance : expectedComponentInstances) {
// verify the expected container exists
int containerId = expectedComponentInstance.first;
InstanceId instanceId = expectedComponentInstance.second;
assertTrue(String.format("Container with id %s not found", containerId), plan.getContainer(containerId).isPresent());
expectedContainerIds.add(containerId);
// and that the instance exists on it
boolean instanceFound = false;
PackingPlan.ContainerPlan containerPlan = plan.getContainer(containerId).get();
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
if (instancePlan.getTaskId() == instanceId.getTaskId()) {
instanceFound = true;
assertEquals("Wrong componentName for task " + instancePlan.getTaskId(), instanceId.getComponentName(), instancePlan.getComponentName());
assertEquals("Wrong getComponentIndex for task " + instancePlan.getTaskId(), instanceId.getComponentIndex(), instancePlan.getComponentIndex());
break;
}
}
assertTrue(String.format("Container (%s) did not include expected instance with taskId %d", containerPlan, instanceId.getTaskId()), instanceFound);
}
Map<Integer, PackingPlan.InstancePlan> taskIds = new HashMap<>();
Map<String, Set<PackingPlan.InstancePlan>> componentInstances = new HashMap<>();
for (PackingPlan.ContainerPlan containerPlan : plan.getContainers()) {
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
// check for taskId collisions
PackingPlan.InstancePlan collisionInstance = taskIds.get(instancePlan.getTaskId());
assertNull(String.format("Task id collision between instance %s and %s", instancePlan, collisionInstance), collisionInstance);
taskIds.put(instancePlan.getTaskId(), instancePlan);
// check for componentIndex collisions
Set<PackingPlan.InstancePlan> instances = componentInstances.get(instancePlan.getComponentName());
if (instances != null) {
for (PackingPlan.InstancePlan instance : instances) {
assertTrue(String.format("Component index collision between instance %s and %s", instance, instancePlan), instance.getComponentIndex() != instancePlan.getComponentIndex());
}
}
componentInstances.computeIfAbsent(instancePlan.getComponentName(), k -> new HashSet<>());
componentInstances.get(instancePlan.getComponentName()).add(instancePlan);
}
}
assertEquals(expectedContainerIds.size(), plan.getContainers().size());
}
Aggregations