Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 1 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doPackingAndScalingTest.

/**
 * Performs a scaling test for a specific topology. It first
 * computes an initial packing plan as a basis for scaling.
 * Given specific component parallelism changes, a new packing plan is produced.
 *
 * @param testTopology Input topology
 * @param componentChanges parallelism changes for scale up/down
 * @param boltRes RAM allocated to bolts
 * @param testBoltParallelism bolt parallelism
 * @param spoutRes RAM allocated to spouts
 * @param testSpoutParallelism spout parallelism
 * @param numContainersBeforeRepack number of containers that the initial packing plan should use
 * @param numContainersAfterRepack number of instances expected before scaling
 * @return the new packing plan
 */
protected PackingPlan doPackingAndScalingTest(TopologyAPI.Topology testTopology, Map<String, Integer> componentChanges, Resource boltRes, int testBoltParallelism, Resource spoutRes, int testSpoutParallelism, int numContainersBeforeRepack, int numContainersAfterRepack, Resource maxContainerResource) {
    PackingPlan packingPlan = doPackingTest(testTopology, boltRes, testBoltParallelism, spoutRes, testSpoutParallelism, numContainersBeforeRepack, maxContainerResource);
    PackingPlan newPackingPlan = doScalingTest(testTopology, packingPlan, componentChanges, boltRes, testBoltParallelism, spoutRes, testSpoutParallelism, numContainersAfterRepack, maxContainerResource);
    return newPackingPlan;
}

Example 2 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doPackingTest.

protected PackingPlan doPackingTest(TopologyAPI.Topology testTopology, Resource boltRes, int testBoltParallelism, Resource spoutRes, int testSpoutParallelism, int testNumContainers, Resource maxContainerResource) {
    PackingPlan packingPlan = pack(testTopology);
    Assert.assertEquals(testNumContainers, packingPlan.getContainers().size());
    Assert.assertEquals(testBoltParallelism + testSpoutParallelism, (int) packingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, testBoltParallelism);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism);
    AssertPacking.assertInstanceRam(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRes.getRam(), spoutRes.getRam());
    AssertPacking.assertInstanceCpu(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRes.getCpu(), spoutRes.getCpu());
    AssertPacking.assertInstanceIndices(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertContainerRam(packingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(packingPlan.getContainers(), maxContainerResource.getCpu());
    return packingPlan;
}

Example 3 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doPackingTestWithPartialResource.

protected PackingPlan doPackingTestWithPartialResource(TopologyAPI.Topology testTopology, Optional<ByteAmount> boltRam, Optional<Double> boltCpu, int testBoltParallelism, Optional<ByteAmount> spoutRam, Optional<Double> spoutCpu, int testSpoutParallelism, int testNumContainers, Resource padding, Resource maxContainerResource) {
    PackingPlan packingPlan = pack(testTopology);
    Assert.assertEquals(testNumContainers, packingPlan.getContainers().size());
    Assert.assertEquals(testBoltParallelism + testSpoutParallelism, (int) packingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, testBoltParallelism);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        int instancesCount = containerPlan.getInstances().size();
        if (!boltRam.isPresent() && !spoutRam.isPresent()) {
            ByteAmount instanceRam = maxContainerResource.getRam().minus(padding.getRam()).divide(instancesCount);
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceRam, instancePlan.getResource().getRam());
            }
        } else if (!boltRam.isPresent() || !spoutRam.isPresent()) {
            String explicitComponent = boltRam.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltRam.isPresent() ? SPOUT_NAME : BOLT_NAME;
            ByteAmount explicitRam = boltRam.orElseGet(spoutRam::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitRam, instancePlan.getResource().getRam());
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals(maxContainerResource.getRam().minus(explicitRam.multiply(explicitCount)).minus(padding.getRam()).divide(implicitCount), instancePlan.getResource().getRam());
                }
            }
        }
        if (!boltCpu.isPresent() && !spoutCpu.isPresent()) {
            double instanceCpu = (maxContainerResource.getCpu() - padding.getCpu()) / instancesCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceCpu, instancePlan.getResource().getCpu(), DELTA);
            }
        } else if (!boltCpu.isPresent() || !spoutCpu.isPresent()) {
            String explicitComponent = boltCpu.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltCpu.isPresent() ? SPOUT_NAME : BOLT_NAME;
            double explicitCpu = boltCpu.orElseGet(spoutCpu::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitCpu, instancePlan.getResource().getCpu(), DELTA);
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals((maxContainerResource.getCpu() - explicitCpu * explicitCount - padding.getCpu()) / implicitCount, instancePlan.getResource().getCpu(), DELTA);
                }
            }
        }
    }
    AssertPacking.assertInstanceIndices(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertContainerRam(packingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(packingPlan.getContainers(), maxContainerResource.getCpu());
    return packingPlan;
}

Example 4 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doScalingTest.

protected PackingPlan doScalingTest(TopologyAPI.Topology testTopology, PackingPlan packingPlan, Map<String, Integer> componentChanges, Resource boltRes, int testBoltParallelism, Resource spoutRes, int testSpoutParallelism, int testNumContainers, Resource maxContainerResource) {
    PackingPlan newPackingPlan = repack(testTopology, packingPlan, componentChanges);
    Assert.assertEquals(testNumContainers, newPackingPlan.getContainers().size());
    AssertPacking.assertInstanceRam(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRes.getRam(), spoutRes.getRam());
    AssertPacking.assertInstanceCpu(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRes.getCpu(), spoutRes.getCpu());
    // AssertPacking.assertInstanceIndices(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, testBoltParallelism + componentChanges.getOrDefault(BOLT_NAME, 0));
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism + componentChanges.getOrDefault(SPOUT_NAME, 0));
    AssertPacking.assertContainerRam(newPackingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(newPackingPlan.getContainers(), maxContainerResource.getCpu());
    return newPackingPlan;
}

Example 5 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanBuilderTest method testInvalidContainerRemoveFromPackingPlan.

@Test(expected = PackingException.class)
public void testInvalidContainerRemoveFromPackingPlan() throws ConstraintViolationException {
    PackingPlan plan = doCreatePackingPlanTest(testContainerInstances);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, String>[] removed = new Pair[] { new Pair<>(7, "componentA") };
    PackingTestHelper.removeFromTestPackingPlan(TOPOLOGY_ID, plan, removed, 0);
}