Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 6 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, 2, containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        int instanceCount = containerPlan.getInstances().size();
        Assert.assertEquals(Math.round(instanceCount * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(instanceCount).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(instanceCount).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 7 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class FirstFitDecreasingPacking method getSortedInstances.

/**
 * Sort the components in decreasing order based on their RAM requirements
 *
 * @return The sorted list of components and their RAM requirements
 */
private List<ResourceRequirement> getSortedInstances(Set<String> componentNames) {
    List<ResourceRequirement> resourceRequirements = new ArrayList<>();
    for (String componentName : componentNames) {
        Resource requiredResource = this.componentResourceMap.getOrDefault(componentName, defaultInstanceResources);
        resourceRequirements.add(new ResourceRequirement(componentName, requiredResource.getRam(), requiredResource.getCpu()));
    }
    Collections.sort(resourceRequirements, sortingStrategy.reversed());
    return resourceRequirements;
}

Example 8 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method buildContainerPlans.

/**
 * Estimate the per instance and topology resources for the packing plan based on the ramMap,
 * instance defaults and paddingPercentage.
 *
 * @return container plans
 */
private static Set<PackingPlan.ContainerPlan> buildContainerPlans(Map<Integer, Container> containerInstances) {
    Set<PackingPlan.ContainerPlan> containerPlans = new LinkedHashSet<>();
    for (Integer containerId : containerInstances.keySet()) {
        Container container = containerInstances.get(containerId);
        if (container.getInstances().size() == 0) {
            continue;
        }
        Resource totalUsedResources = container.getTotalUsedResources();
        Resource resource = new Resource(Math.round(totalUsedResources.getCpu()), totalUsedResources.getRam(), totalUsedResources.getDisk());
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, container.getInstances(), resource);
        containerPlans.add(containerPlan);
    }
    return containerPlans;
}

Example 9 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method addInstance.

// adds an instance to a container with id containerId. If that container does not exist, it will
// be lazily initialized, which could result in more containers than those requested using the
// updateNumContainers method
public PackingPlanBuilder addInstance(Integer containerId, String componentName) throws ConstraintViolationException {
    // create container if not existed
    initContainer(containerId);
    Integer taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
    Integer componentIndex = componentIndexes.containsKey(componentName) ? componentIndexes.get(componentName).last() + 1 : 0;
    InstanceId instanceId = new InstanceId(componentName, taskId, componentIndex);
    Resource instanceResource = componentResourceMap.getOrDefault(componentName, defaultInstanceResource);
    Container container = containers.get(containerId);
    PackingPlan.InstancePlan instancePlan = new PackingPlan.InstancePlan(instanceId, instanceResource);
    // Check constraints
    for (InstanceConstraint constraint : instanceConstraints) {
        constraint.validate(instancePlan);
    }
    for (PackingConstraint constraint : packingConstraints) {
        constraint.validate(container, instancePlan);
    }
    addToContainer(container, instancePlan, this.componentIndexes, this.taskIds);
    LOG.finest(String.format("Added to container %d instance %s", containerId, instanceId));
    return this;
}

Example 10 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class V1Controller method submit.

/**
 * Configures all components required by a <code>topology</code> and submits it to the Kubernetes scheduler.
 * @param packingPlan Used to configure the StatefulSets <code>Resource</code>s and replica count.
 * @return Success indicator.
 */
@Override
boolean submit(PackingPlan packingPlan) {
    final String topologyName = getTopologyName();
    if (!topologyName.equals(topologyName.toLowerCase())) {
        throw new TopologySubmissionException("K8S scheduler does not allow upper case topology's.");
    }
    final Resource containerResource = getContainerResource(packingPlan);
    final V1Service topologyService = createTopologyService();
    try {
        coreClient.createNamespacedService(getNamespace(), topologyService, null, null, null);
    } catch (ApiException e) {
        KubernetesUtils.logExceptionWithDetails(LOG, "Error creating topology service", e);
        throw new TopologySubmissionException(e.getMessage());
    }
    // Find the max number of instances in a container so that we can open
    // enough ports if remote debugging is enabled.
    int numberOfInstances = 0;
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        numberOfInstances = Math.max(numberOfInstances, containerPlan.getInstances().size());
    }
    final V1StatefulSet executors = createStatefulSet(containerResource, numberOfInstances, true);
    final V1StatefulSet manager = createStatefulSet(containerResource, numberOfInstances, false);
    try {
        appsClient.createNamespacedStatefulSet(getNamespace(), executors, null, null, null);
        appsClient.createNamespacedStatefulSet(getNamespace(), manager, null, null, null);
    } catch (ApiException e) {
        final String message = String.format("Error creating topology: %s%n", e.getResponseBody());
        KubernetesUtils.logExceptionWithDetails(LOG, message, e);
        throw new TopologySubmissionException(message);
    }
    return true;
}