Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 61 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanBuilderTest method testInvalidComponentRemoveFromPackingPlan.

@Test(expected = PackingException.class)
public void testInvalidComponentRemoveFromPackingPlan() throws ConstraintViolationException {
    PackingPlan plan = doCreatePackingPlanTest(testContainerInstances);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, String>[] removed = new Pair[] { new Pair<>(1, "componentC") };
    PackingTestHelper.removeFromTestPackingPlan(TOPOLOGY_ID, plan, removed, 0);
}

Example 62 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanBuilderTest method testGetContainers.

/**
 * Tests the getContainers method.
 */
@Test
public void testGetContainers() {
    Resource padding = new Resource(1.0, ByteAmount.fromGigabytes(1), ByteAmount.fromGigabytes(1));
    Map<Integer, List<InstanceId>> packing = new HashMap<>();
    packing.put(7, Arrays.asList(new InstanceId("spout", 1, 0), new InstanceId("bolt", 2, 0)));
    packing.put(3, Arrays.asList(new InstanceId("spout", 3, 0), new InstanceId("bolt", 4, 0)));
    PackingPlan packingPlan = generatePacking(packing);
    Map<Integer, Container> containers = PackingPlanBuilder.getContainers(packingPlan, packingPlan.getMaxContainerResources(), padding, new HashMap<String, TreeSet<Integer>>(), new TreeSet<Integer>());
    assertEquals(packing.size(), containers.size());
    for (Integer containerId : packing.keySet()) {
        Container foundContainer = containers.get(containerId);
        assertEquals(padding, foundContainer.getPadding());
        assertEquals(packingPlan.getMaxContainerResources(), foundContainer.getCapacity());
        assertEquals(2, foundContainer.getInstances().size());
    }
}

Example 63 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanBuilderTest method testExceededCapacityAddingToPackingPlan.

@Test(expected = ResourceExceededException.class)
public void testExceededCapacityAddingToPackingPlan() throws ConstraintViolationException {
    PackingPlan plan = doCreatePackingPlanTest(testContainerInstances);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] added = new Pair[] { new Pair<>(3, new InstanceId("componentB", 4, 1)), new Pair<>(3, new InstanceId("componentB", 5, 2)) };
    PackingTestHelper.addToTestPackingPlan(TOPOLOGY_ID, plan, PackingTestHelper.toContainerIdComponentNames(added), 0);
}

Example 64 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class ResourceCompliantRRPackingTest method testContainerRequestedResourcesSingleContainer.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResourcesSingleContainer() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.setNumStmgrs(numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, numContainers, containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        Assert.assertEquals(Math.round(totalInstances * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 65 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class CommonPackingTests method doScalingTestWithPartialResource.

protected PackingPlan doScalingTestWithPartialResource(TopologyAPI.Topology testTopology, PackingPlan packingPlan, Map<String, Integer> componentChanges, Optional<ByteAmount> boltRam, Optional<Double> boltCpu, int testBoltParallelism, Optional<ByteAmount> spoutRam, Optional<Double> spoutCpu, int testSpoutParallelism, int testNumContainers, Resource padding, Resource maxContainerResource) {
    System.out.println(packingPlan);
    PackingPlan newPackingPlan = repack(testTopology, packingPlan, componentChanges);
    System.out.println(newPackingPlan);
    Assert.assertEquals(testNumContainers, newPackingPlan.getContainers().size());
    Assert.assertEquals(testBoltParallelism + testSpoutParallelism + componentChanges.getOrDefault(BOLT_NAME, 0) + componentChanges.getOrDefault(SPOUT_NAME, 0), (int) newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, testBoltParallelism + componentChanges.getOrDefault(BOLT_NAME, 0));
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism + componentChanges.getOrDefault(SPOUT_NAME, 0));
    for (PackingPlan.ContainerPlan containerPlan : newPackingPlan.getContainers()) {
        int instancesCount = containerPlan.getInstances().size();
        if (!boltRam.isPresent() && !spoutRam.isPresent()) {
            ByteAmount instanceRam = maxContainerResource.getRam().minus(padding.getRam()).divide(instancesCount);
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceRam, instancePlan.getResource().getRam());
            }
        } else if (!boltRam.isPresent() || !spoutRam.isPresent()) {
            String explicitComponent = boltRam.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltRam.isPresent() ? SPOUT_NAME : BOLT_NAME;
            ByteAmount explicitRam = boltRam.orElseGet(spoutRam::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitRam, instancePlan.getResource().getRam());
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals(maxContainerResource.getRam().minus(explicitRam.multiply(explicitCount)).minus(padding.getRam()).divide(implicitCount), instancePlan.getResource().getRam());
                }
            }
        }
        if (!boltCpu.isPresent() && !spoutCpu.isPresent()) {
            double instanceCpu = (maxContainerResource.getCpu() - padding.getCpu()) / instancesCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceCpu, instancePlan.getResource().getCpu(), DELTA);
            }
        } else if (!boltCpu.isPresent() || !spoutCpu.isPresent()) {
            String explicitComponent = boltCpu.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltCpu.isPresent() ? SPOUT_NAME : BOLT_NAME;
            double explicitCpu = boltCpu.orElseGet(spoutCpu::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitCpu, instancePlan.getResource().getCpu(), DELTA);
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals((maxContainerResource.getCpu() - explicitCpu * explicitCount - padding.getCpu()) / implicitCount, instancePlan.getResource().getCpu(), DELTA);
                }
            }
        }
    }
    AssertPacking.assertInstanceIndices(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertContainerRam(newPackingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(newPackingPlan.getContainers(), maxContainerResource.getCpu());
    return newPackingPlan;
}