Examples with PackingPlanBuilder com.twitter.heron.packing.builder.PackingPlanBuilder used on opensource projects

Example 1 with PackingPlanBuilder

use of com.twitter.heron.packing.builder.PackingPlanBuilder in project heron by twitter.

the class ResourceCompliantRRPacking method pack.

@Override
public PackingPlan pack() {
    while (true) {
        try {
            PackingPlanBuilder planBuilder = newPackingPlanBuilder(null);
            planBuilder.updateNumContainers(numContainers);
            planBuilder = getResourceCompliantRRAllocation(planBuilder);
            return planBuilder.build();
        } catch (ResourceExceededException e) {
            //Not enough containers. Adjust the number of containers.
            LOG.finest(String.format("%s Increasing the number of containers to %s and attempting to place again.", e.getMessage(), this.numContainers + 1));
            increaseNumContainers(1);
            resetToFirstContainer();
        }
    }
}

Example 2 with PackingPlanBuilder

use of com.twitter.heron.packing.builder.PackingPlanBuilder in project heron by twitter.

the class ResourceCompliantRRPacking method repack.

/**
   * Get a new packing plan given an existing packing plan and component-level changes.
   *
   * @return new packing plan
   */
@Override
public PackingPlan repack(PackingPlan currentPackingPlan, Map<String, Integer> componentChanges) {
    this.numContainers = currentPackingPlan.getContainers().size();
    resetToFirstContainer();
    int additionalContainers = computeNumAdditionalContainers(componentChanges, currentPackingPlan);
    if (additionalContainers > 0) {
        increaseNumContainers(additionalContainers);
        LOG.info(String.format("Allocated %s additional containers for repack bring the number of containers to %s.", additionalContainers, this.numContainers));
    }
    while (true) {
        try {
            PackingPlanBuilder planBuilder = newPackingPlanBuilder(currentPackingPlan);
            planBuilder.updateNumContainers(numContainers);
            planBuilder = getResourceCompliantRRAllocation(planBuilder, componentChanges);
            return planBuilder.build();
        } catch (ResourceExceededException e) {
            //Not enough containers. Adjust the number of containers.
            increaseNumContainers(1);
            resetToFirstContainer();
            LOG.info(String.format("%s Increasing the number of containers to %s and attempting packing again.", e.getMessage(), this.numContainers));
        }
    }
}

Example 3 with PackingPlanBuilder

use of com.twitter.heron.packing.builder.PackingPlanBuilder in project heron by twitter.

the class PackingTestHelper method generateTestPackingPlan.

/**
   * Returns a PackingPlan to use for testing scale up/down that has instances of specific
   * components on specific containers.
   */
private static PackingPlan generateTestPackingPlan(String topologyName, PackingPlan previousPackingPlan, Pair<Integer, String>[] addInstances, Pair<Integer, String>[] removeInstances, int containerPadding) throws ResourceExceededException {
    PackingPlanBuilder builder = new PackingPlanBuilder(topologyName, previousPackingPlan);
    int instanceCount = 0;
    if (previousPackingPlan != null) {
        instanceCount = previousPackingPlan.getInstanceCount();
    } else if (addInstances != null) {
        instanceCount = addInstances.length;
    }
    // use basic default resource to allow all instances to fit on a single container, if that's
    // what the tester desired. We can extend this to permit passing custom resource requirements
    // as needed.
    builder.setDefaultInstanceResource(new Resource(1, ByteAmount.fromMegabytes(192), ByteAmount.fromMegabytes(1)));
    builder.setMaxContainerResource(new Resource(instanceCount, ByteAmount.fromMegabytes(192).multiply(instanceCount), ByteAmount.fromMegabytes(instanceCount)));
    // This setting is important, see https://github.com/twitter/heron/issues/1577
    builder.setRequestedContainerPadding(containerPadding);
    if (addInstances != null) {
        for (Pair<Integer, String> componentInstance : addInstances) {
            builder.addInstance(componentInstance.first, componentInstance.second);
        }
    }
    if (removeInstances != null) {
        for (Pair<Integer, String> componentInstance : removeInstances) {
            builder.removeInstance(componentInstance.first, componentInstance.second);
        }
    }
    return builder.build();
}

Example 4 with PackingPlanBuilder

use of com.twitter.heron.packing.builder.PackingPlanBuilder in project incubator-heron by apache.

the class ResourceCompliantRRPacking method repack.

/**
 * Get a new packing plan given an existing packing plan and component-level changes.
 *
 * @return new packing plan
 */
@Override
public PackingPlan repack(PackingPlan currentPackingPlan, Map<String, Integer> componentChanges) {
    this.numContainers = currentPackingPlan.getContainers().size();
    resetToFirstContainer();
    int additionalContainers = computeNumAdditionalContainers(componentChanges, currentPackingPlan);
    if (additionalContainers > 0) {
        increaseNumContainers(additionalContainers);
        LOG.info(String.format("Allocated %s additional containers for repack bring the number of containers to %s.", additionalContainers, this.numContainers));
    }
    while (true) {
        try {
            PackingPlanBuilder planBuilder = newPackingPlanBuilder(currentPackingPlan);
            planBuilder.updateNumContainers(numContainers);
            planBuilder = getResourceCompliantRRAllocation(planBuilder, componentChanges);
            return planBuilder.build();
        } catch (ResourceExceededException e) {
            // Not enough containers. Adjust the number of containers.
            increaseNumContainers(1);
            resetToFirstContainer();
            LOG.info(String.format("%s Increasing the number of containers to %s and attempting packing again.", e.getMessage(), this.numContainers));
        }
    }
}

Example 5 with PackingPlanBuilder

use of com.twitter.heron.packing.builder.PackingPlanBuilder in project incubator-heron by apache.

the class PackingTestHelper method generateTestPackingPlan.

/**
 * Returns a PackingPlan to use for testing scale up/down that has instances of specific
 * components on specific containers.
 */
private static PackingPlan generateTestPackingPlan(String topologyName, PackingPlan previousPackingPlan, Pair<Integer, String>[] addInstances, Pair<Integer, String>[] removeInstances, int containerPadding) throws ResourceExceededException {
    PackingPlanBuilder builder = new PackingPlanBuilder(topologyName, previousPackingPlan);
    int instanceCount = 0;
    if (previousPackingPlan != null) {
        instanceCount = previousPackingPlan.getInstanceCount();
    } else if (addInstances != null) {
        instanceCount = addInstances.length;
    }
    // use basic default resource to allow all instances to fit on a single container, if that's
    // what the tester desired. We can extend this to permit passing custom resource requirements
    // as needed.
    builder.setDefaultInstanceResource(new Resource(1, ByteAmount.fromMegabytes(192), ByteAmount.fromMegabytes(1)));
    builder.setMaxContainerResource(new Resource(instanceCount, ByteAmount.fromMegabytes(192).multiply(instanceCount), ByteAmount.fromMegabytes(instanceCount)));
    // This setting is important, see https://github.com/twitter/heron/issues/1577
    builder.setRequestedContainerPadding(containerPadding);
    if (addInstances != null) {
        for (Pair<Integer, String> componentInstance : addInstances) {
            builder.addInstance(componentInstance.first, componentInstance.second);
        }
    }
    if (removeInstances != null) {
        for (Pair<Integer, String> componentInstance : removeInstances) {
            builder.removeInstance(componentInstance.first, componentInstance.second);
        }
    }
    return builder.build();
}