Examples with ByteAmount org.apache.heron.common.basics.ByteAmount used on opensource projects

Example 6 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testContainerRequestedResourcesWhenRamPaddingSet.

/**
 * Test the scenario container level resource config are set
 */
@Test
public void testContainerRequestedResourcesWhenRamPaddingSet() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount containerRamPadding = ByteAmount.fromMegabytes(512);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setContainerRamPadding(containerRamPadding);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.empty(), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding().cloneWithRam(containerRamPadding), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(containerRamPadding).divide(instancesCount), resources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, resources.iterator().next().getCpu(), DELTA);
    }
}

Example 7 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testFullRamMapWithoutContainerRequestedResources.

@Test
public void testFullRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount spoutRam = ByteAmount.fromMegabytes(500);
    ByteAmount boltRam = ByteAmount.fromMegabytes(1000);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.of(spoutRam), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        // Bolt and spout ram sizes are both fixed.
        Assert.assertEquals(2, differentResources.size());
    }
}

Example 8 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested.

/**
 * Test the scenario RAM map config is fully set
 */
@Test
public void testCompleteRamMapRequested() throws Exception {
    // Explicit set max resources for container
    // the value should be ignored, since we set the complete component RAM map
    ByteAmount containerRam = ByteAmount.fromGigabytes(15);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources.cloneWithRam(spoutRam), spoutParallelism, 2, containerResource);
}

Example 9 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, 2, containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        int instanceCount = containerPlan.getInstances().size();
        Assert.assertEquals(Math.round(instanceCount * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(instanceCount).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(instanceCount).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 10 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method scaleDownAndUpWithExtraPadding.

/**
 * Test the scenario where scaling down and up is simultaneously requested and padding is
 * configured
 */
@Test
public void scaleDownAndUpWithExtraPadding() throws Exception {
    int paddingPercentage = 50;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    boltParallelism = 2;
    spoutParallelism = 1;
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int spoutScalingUp = 1;
    int boltScalingDown = -2;
    Map<String, Integer> componentChanges = new HashMap<>();
    // 2 spouts
    componentChanges.put(SPOUT_NAME, spoutScalingUp);
    // 0 bolts
    componentChanges.put(BOLT_NAME, boltScalingDown);
    int numContainersBeforeRepack = 2;
    int numContainersAfterRepack = 1;
    doPackingAndScalingTest(topology, componentChanges, instanceDefaultResources, boltParallelism, instanceDefaultResources.cloneWithRam(spoutRam), spoutParallelism, numContainersBeforeRepack, numContainersAfterRepack, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}