Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 26 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RoundRobinPackingTest method testCompleteRamMapRequested.

/**
   * Test the scenario ram map config is partially set
   */
@Test
public void testCompleteRamMapRequested() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}

Example 27 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doScaleDownTest.

private void doScaleDownTest(Pair<Integer, InstanceId>[] initialComponentInstances, Map<String, Integer> componentChanges, Pair<Integer, InstanceId>[] expectedComponentInstances) throws ResourceExceededException {
    String topologyId = this.topology.getId();
    // The padding percentage used in repack() must be <= one as used in pack(), otherwise we can't
    // reconstruct the PackingPlan, see https://github.com/twitter/heron/issues/1577
    PackingPlan initialPackingPlan = PackingTestHelper.addToTestPackingPlan(topologyId, null, PackingTestHelper.toContainerIdComponentNames(initialComponentInstances), DEFAULT_CONTAINER_PADDING);
    AssertPacking.assertPackingPlan(topologyId, initialComponentInstances, initialPackingPlan);
    PackingPlan newPackingPlan = repack(this.topology, initialPackingPlan, componentChanges);
    AssertPacking.assertPackingPlan(topologyId, expectedComponentInstances, newPackingPlan);
}

Example 28 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doScalingTest.

/**
   * Performs a scaling test for a specific topology. It first
   * computes an initial packing plan as a basis for scaling.
   * Given specific component parallelism changes, a new packing plan is produced.
   *
   * @param testTopology Input topology
   * @param componentChanges parallelism changes for scale up/down
   * @param boltRam ram allocated to bolts
   * @param testBoltParallelism bolt parallelism
   * @param spoutRam ram allocated to spouts
   * @param testSpoutParallelism spout parallelism
   * @param numContainersBeforeRepack number of containers that the initial packing plan should use
   * @param totalInstancesExpected number of instances expected before scaling
   * @return the new packing plan
   */
protected PackingPlan doScalingTest(TopologyAPI.Topology testTopology, Map<String, Integer> componentChanges, ByteAmount boltRam, int testBoltParallelism, ByteAmount spoutRam, int testSpoutParallelism, int numContainersBeforeRepack, int totalInstancesExpected) {
    PackingPlan packingPlan = pack(testTopology);
    Assert.assertEquals(numContainersBeforeRepack, packingPlan.getContainers().size());
    Assert.assertEquals(totalInstancesExpected, (int) packingPlan.getInstanceCount());
    AssertPacking.assertContainers(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, null);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, testBoltParallelism);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism);
    PackingPlan newPackingPlan = repack(testTopology, packingPlan, componentChanges);
    AssertPacking.assertContainerRam(newPackingPlan.getContainers(), packingPlan.getMaxContainerResources().getRam());
    return newPackingPlan;
}

Example 29 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method testInvalidRamInstance.

/**
   * Test invalid ram for instance
   */
@Test(expected = PackingException.class)
public void testInvalidRamInstance() throws Exception {
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    int defaultNumInstancesperContainer = 4;
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.ZERO;
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer));
}

Example 30 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class FirstFitDecreasingPackingTest method removeFirstContainer.

/**
   * Test the scenario where scaling down is requested and the first container is removed.
   */
@Test
public void removeFirstContainer() throws Exception {
    /* The packing plan consists of two containers. The first one contains 4 spouts and
       the second one contains 3 bolts. During scaling we remove 4 spouts and thus the f
       first container is removed.
     */
    int spoutScalingDown = -4;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(SPOUT_NAME, spoutScalingDown);
    int numContainersBeforeRepack = 2;
    PackingPlan newPackingPlan = doDefaultScalingTest(componentChanges, numContainersBeforeRepack);
    Assert.assertEquals(1, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (totalInstances + spoutScalingDown), newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, 0);
}