Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 36 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtils method computeTotalResourceChange.

/**
 * Identifies the resources reclaimed by the components that will be scaled down
 *
 * @return Total resources reclaimed
 */
public static Resource computeTotalResourceChange(TopologyAPI.Topology topology, Map<String, Integer> componentChanges, Resource defaultInstanceResources, ScalingDirection scalingDirection) {
    double cpu = 0;
    ByteAmount ram = ByteAmount.ZERO;
    ByteAmount disk = ByteAmount.ZERO;
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<String, Integer> componentsToScale = PackingUtils.getComponentsToScale(componentChanges, scalingDirection);
    for (String component : componentsToScale.keySet()) {
        int parallelismChange = Math.abs(componentChanges.get(component));
        cpu += parallelismChange * defaultInstanceResources.getCpu();
        disk = disk.plus(defaultInstanceResources.getDisk().multiply(parallelismChange));
        if (ramMap.containsKey(component)) {
            ram = ram.plus(ramMap.get(component).multiply(parallelismChange));
        } else {
            ram = ram.plus(defaultInstanceResources.getRam().multiply(parallelismChange));
        }
    }
    return new Resource(cpu, ram, disk);
}

Example 37 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtils method finalizePadding.

/**
 * Finalize padding by taking Math.max(containerResource * paddingPercent, paddingValue)
 *
 * @param containerResource max container resource
 * @param padding padding value
 * @param paddingPercentage padding percent
 * @return finalized padding amount
 */
public static Resource finalizePadding(Resource containerResource, Resource padding, int paddingPercentage) {
    double cpuPadding = Math.max(padding.getCpu(), containerResource.getCpu() * paddingPercentage / 100);
    ByteAmount ramPadding = ByteAmount.fromBytes(Math.max(padding.getRam().asBytes(), containerResource.getRam().asBytes() * paddingPercentage / 100));
    ByteAmount diskPadding = ByteAmount.fromBytes(Math.max(padding.getDisk().asBytes(), containerResource.getDisk().asBytes() * paddingPercentage / 100));
    return new Resource(cpuPadding, ramPadding, diskPadding);
}

Example 38 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method getContainers.

/**
 * Generates the containers that correspond to the current packing plan
 * along with their associated instances.
 *
 * @return Map of containers for the current packing plan, keyed by containerId
 */
@VisibleForTesting
static Map<Integer, Container> getContainers(PackingPlan currentPackingPlan, Resource maxContainerResource, Resource padding, Map<String, TreeSet<Integer>> componentIndexes, TreeSet<Integer> taskIds) {
    Map<Integer, Container> containers = new HashMap<>();
    Resource capacity = maxContainerResource;
    for (PackingPlan.ContainerPlan currentContainerPlan : currentPackingPlan.getContainers()) {
        Container container = new Container(currentContainerPlan.getId(), capacity, padding);
        for (PackingPlan.InstancePlan instancePlan : currentContainerPlan.getInstances()) {
            addToContainer(container, instancePlan, componentIndexes, taskIds);
        }
        containers.put(currentContainerPlan.getId(), container);
    }
    return containers;
}

Example 39 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method initContainers.

private void initContainers() {
    assertResourceSettings();
    if (this.componentIndexes == null) {
        this.componentIndexes = new HashMap<>();
    }
    if (this.taskIds == null) {
        this.taskIds = new TreeSet<>();
    }
    // if this is the first time called, initialize container map with empty or existing containers
    if (this.containers == null) {
        if (this.existingPacking == null) {
            this.containers = new HashMap<>();
            for (int containerId = 1; containerId <= numContainers; containerId++) {
                this.containers.put(containerId, new Container(containerId, this.maxContainerResource, this.requestedContainerPadding));
            }
        } else {
            this.containers = getContainers(this.existingPacking, this.maxContainerResource, this.requestedContainerPadding, this.componentIndexes, this.taskIds);
        }
    }
    if (this.numContainers > this.containers.size()) {
        List<Scorer<Container>> scorers = new ArrayList<>();
        scorers.add(new ContainerIdScorer());
        List<Container> sortedContainers = sortContainers(scorers, this.containers.values());
        int nextContainerId = sortedContainers.get(sortedContainers.size() - 1).getContainerId() + 1;
        Resource capacity = this.containers.get(sortedContainers.get(0).getContainerId()).getCapacity();
        for (int i = 0; i < numContainers - this.containers.size(); i++) {
            this.containers.put(nextContainerId, new Container(nextContainerId, capacity, this.requestedContainerPadding));
            nextContainerId++;
        }
    }
}

Example 40 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class ScorerTest method init.

@Before
public void init() throws ResourceExceededException {
    Resource containerCapacity = new Resource(1000, ByteAmount.fromGigabytes(100), ByteAmount.fromGigabytes(100));
    testContainers = new Container[] { new Container(1, containerCapacity, Resource.EMPTY_RESOURCE), new Container(3, containerCapacity, Resource.EMPTY_RESOURCE), new Container(4, containerCapacity, Resource.EMPTY_RESOURCE) };
    addInstance(testContainers[0], "A", 0);
    addInstance(testContainers[0], "A", 1);
    addInstance(testContainers[0], "B", 2);
    addInstance(testContainers[0], "B", 3);
    addInstance(testContainers[1], "A", 4);
    addInstance(testContainers[1], "B", 5);
    addInstance(testContainers[1], "B", 6);
    addInstance(testContainers[2], "A", 7);
    addInstance(testContainers[2], "A", 8);
}