Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 31 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtilsTest method testResourceScaleDown.

@Test
public void testResourceScaleDown() {
    int noSpouts = 6;
    int noBolts = 3;
    int boltScalingDown = 2;
    org.apache.heron.api.Config topologyConfig = new org.apache.heron.api.Config();
    TopologyAPI.Topology topology = getTopology(noSpouts, noBolts, topologyConfig);
    Config config = PackingTestUtils.newTestConfig(topology);
    Resource defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    Map<String, Integer> componentChanges = new HashMap<>();
    // 1 bolt
    componentChanges.put("bolt", -boltScalingDown);
    Resource scaledownResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.DOWN);
    Assert.assertEquals((long) (boltScalingDown * defaultInstanceResources.getCpu()), (long) scaledownResource.getCpu());
    Assert.assertEquals(defaultInstanceResources.getRam().multiply(boltScalingDown), scaledownResource.getRam());
    Assert.assertEquals(defaultInstanceResources.getDisk().multiply(boltScalingDown), scaledownResource.getDisk());
}

Example 32 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilderTest method generatePacking.

private static PackingPlan generatePacking(Map<Integer, List<InstanceId>> basePacking) throws RuntimeException {
    Resource resource = new Resource(2.0, ByteAmount.fromGigabytes(6), ByteAmount.fromGigabytes(25));
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : basePacking.keySet()) {
        List<InstanceId> instanceList = basePacking.get(containerId);
        Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
        for (InstanceId instanceId : instanceList) {
            String componentName = instanceId.getComponentName();
            Resource instanceResource;
            switch(componentName) {
                case "bolt":
                    instanceResource = new Resource(1.0, ByteAmount.fromGigabytes(2), ByteAmount.fromGigabytes(10));
                    break;
                case "spout":
                    instanceResource = new Resource(1.0, ByteAmount.fromGigabytes(3), ByteAmount.fromGigabytes(10));
                    break;
                default:
                    throw new RuntimeException(String.format("%s is not a valid component name", componentName));
            }
            instancePlans.add(new PackingPlan.InstancePlan(instanceId, instanceResource));
        }
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
        containerPlans.add(containerPlan);
    }
    return new PackingPlan("", containerPlans);
}

Example 33 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilderTest method testGetContainers.

/**
 * Tests the getContainers method.
 */
@Test
public void testGetContainers() {
    Resource padding = new Resource(1.0, ByteAmount.fromGigabytes(1), ByteAmount.fromGigabytes(1));
    Map<Integer, List<InstanceId>> packing = new HashMap<>();
    packing.put(7, Arrays.asList(new InstanceId("spout", 1, 0), new InstanceId("bolt", 2, 0)));
    packing.put(3, Arrays.asList(new InstanceId("spout", 3, 0), new InstanceId("bolt", 4, 0)));
    PackingPlan packingPlan = generatePacking(packing);
    Map<Integer, Container> containers = PackingPlanBuilder.getContainers(packingPlan, packingPlan.getMaxContainerResources(), padding, new HashMap<String, TreeSet<Integer>>(), new TreeSet<Integer>());
    assertEquals(packing.size(), containers.size());
    for (Integer containerId : packing.keySet()) {
        Container foundContainer = containers.get(containerId);
        assertEquals(padding, foundContainer.getPadding());
        assertEquals(packingPlan.getMaxContainerResources(), foundContainer.getCapacity());
        assertEquals(2, foundContainer.getInstances().size());
    }
}

Example 34 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPackingTest method testContainerRequestedResourcesSingleContainer.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResourcesSingleContainer() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.setNumStmgrs(numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, numContainers, containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        Assert.assertEquals(Math.round(totalInstances * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 35 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPacking method computeNumAdditionalContainers.

/**
 * Computes the additional number of containers needed to accommodate a scale up/down operation
 *
 * @param componentChanges parallelism changes for scale up/down
 * @param packingPlan existing packing plan
 * @return additional number of containers needed
 */
private int computeNumAdditionalContainers(Map<String, Integer> componentChanges, PackingPlan packingPlan) {
    Resource scaleDownResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.DOWN);
    Resource scaleUpResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.UP);
    Resource additionalResource = scaleUpResource.subtractAbsolute(scaleDownResource);
    return (int) additionalResource.divideBy(packingPlan.getMaxContainerResources());
}