use of org.apache.heron.spi.packing.Resource in project heron by twitter.
the class FirstFitDecreasingPackingTest method testContainerRequestedResources.
/**
* Test the scenario where container level resource config are set
*/
@Test
public void testContainerRequestedResources() throws Exception {
// Explicit set resources for container
ByteAmount containerRam = ByteAmount.fromGigabytes(10);
ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
double containerCpu = 30;
Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setContainerDiskRequested(containerDisk);
topologyConfig.setContainerCpuRequested(containerCpu);
topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, 2, containerResource);
for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
int instanceCount = containerPlan.getInstances().size();
Assert.assertEquals(Math.round(instanceCount * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
Assert.assertEquals(instanceDefaultResources.getRam().multiply(instanceCount).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
Assert.assertEquals(instanceDefaultResources.getDisk().multiply(instanceCount).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
// All instances' resource requirement should be equal
// So the size of set should be 1
Set<Resource> resources = new HashSet<>();
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
resources.add(instancePlan.getResource());
}
Assert.assertEquals(1, resources.size());
Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
}
}
use of org.apache.heron.spi.packing.Resource in project heron by twitter.
the class FirstFitDecreasingPacking method getSortedInstances.
/**
* Sort the components in decreasing order based on their RAM requirements
*
* @return The sorted list of components and their RAM requirements
*/
private List<ResourceRequirement> getSortedInstances(Set<String> componentNames) {
List<ResourceRequirement> resourceRequirements = new ArrayList<>();
for (String componentName : componentNames) {
Resource requiredResource = this.componentResourceMap.getOrDefault(componentName, defaultInstanceResources);
resourceRequirements.add(new ResourceRequirement(componentName, requiredResource.getRam(), requiredResource.getCpu()));
}
Collections.sort(resourceRequirements, sortingStrategy.reversed());
return resourceRequirements;
}
use of org.apache.heron.spi.packing.Resource in project heron by twitter.
the class PackingPlanBuilder method buildContainerPlans.
/**
* Estimate the per instance and topology resources for the packing plan based on the ramMap,
* instance defaults and paddingPercentage.
*
* @return container plans
*/
private static Set<PackingPlan.ContainerPlan> buildContainerPlans(Map<Integer, Container> containerInstances) {
Set<PackingPlan.ContainerPlan> containerPlans = new LinkedHashSet<>();
for (Integer containerId : containerInstances.keySet()) {
Container container = containerInstances.get(containerId);
if (container.getInstances().size() == 0) {
continue;
}
Resource totalUsedResources = container.getTotalUsedResources();
Resource resource = new Resource(Math.round(totalUsedResources.getCpu()), totalUsedResources.getRam(), totalUsedResources.getDisk());
PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, container.getInstances(), resource);
containerPlans.add(containerPlan);
}
return containerPlans;
}
use of org.apache.heron.spi.packing.Resource in project heron by twitter.
the class PackingPlanBuilder method addInstance.
// adds an instance to a container with id containerId. If that container does not exist, it will
// be lazily initialized, which could result in more containers than those requested using the
// updateNumContainers method
public PackingPlanBuilder addInstance(Integer containerId, String componentName) throws ConstraintViolationException {
// create container if not existed
initContainer(containerId);
Integer taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
Integer componentIndex = componentIndexes.containsKey(componentName) ? componentIndexes.get(componentName).last() + 1 : 0;
InstanceId instanceId = new InstanceId(componentName, taskId, componentIndex);
Resource instanceResource = componentResourceMap.getOrDefault(componentName, defaultInstanceResource);
Container container = containers.get(containerId);
PackingPlan.InstancePlan instancePlan = new PackingPlan.InstancePlan(instanceId, instanceResource);
// Check constraints
for (InstanceConstraint constraint : instanceConstraints) {
constraint.validate(instancePlan);
}
for (PackingConstraint constraint : packingConstraints) {
constraint.validate(container, instancePlan);
}
addToContainer(container, instancePlan, this.componentIndexes, this.taskIds);
LOG.finest(String.format("Added to container %d instance %s", containerId, instanceId));
return this;
}
use of org.apache.heron.spi.packing.Resource in project heron by twitter.
the class V1Controller method submit.
/**
* Configures all components required by a <code>topology</code> and submits it to the Kubernetes scheduler.
* @param packingPlan Used to configure the StatefulSets <code>Resource</code>s and replica count.
* @return Success indicator.
*/
@Override
boolean submit(PackingPlan packingPlan) {
final String topologyName = getTopologyName();
if (!topologyName.equals(topologyName.toLowerCase())) {
throw new TopologySubmissionException("K8S scheduler does not allow upper case topology's.");
}
final Resource containerResource = getContainerResource(packingPlan);
final V1Service topologyService = createTopologyService();
try {
coreClient.createNamespacedService(getNamespace(), topologyService, null, null, null);
} catch (ApiException e) {
KubernetesUtils.logExceptionWithDetails(LOG, "Error creating topology service", e);
throw new TopologySubmissionException(e.getMessage());
}
// Find the max number of instances in a container so that we can open
// enough ports if remote debugging is enabled.
int numberOfInstances = 0;
for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
numberOfInstances = Math.max(numberOfInstances, containerPlan.getInstances().size());
}
final V1StatefulSet executors = createStatefulSet(containerResource, numberOfInstances, true);
final V1StatefulSet manager = createStatefulSet(containerResource, numberOfInstances, false);
try {
appsClient.createNamespacedStatefulSet(getNamespace(), executors, null, null, null);
appsClient.createNamespacedStatefulSet(getNamespace(), manager, null, null, null);
} catch (ApiException e) {
final String message = String.format("Error creating topology: %s%n", e.getResponseBody());
KubernetesUtils.logExceptionWithDetails(LOG, message, e);
throw new TopologySubmissionException(message);
}
return true;
}
Aggregations