Search in sources :

Example 41 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testHugePartialRamMapWithoutContainerRequestedResources.

// Throw an error if default container resource (default instance resource * number of instances
// + padding) is not enough.
@Test(expected = PackingException.class)
public void testHugePartialRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    ByteAmount boltRam = ByteAmount.fromGigabytes(10);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, differentResources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), differentResources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, differentResources.iterator().next().getCpu(), DELTA);
    }
}
Also used : ByteAmount(org.apache.heron.common.basics.ByteAmount) PackingPlan(org.apache.heron.spi.packing.PackingPlan) Resource(org.apache.heron.spi.packing.Resource) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 42 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testNoRamMapWithoutContainerRequestedResources.

@Test
public void testNoRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Container RAM is not set in config
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.empty(), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, differentResources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), differentResources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, differentResources.iterator().next().getCpu(), DELTA);
    }
}
Also used : ByteAmount(org.apache.heron.common.basics.ByteAmount) PackingPlan(org.apache.heron.spi.packing.PackingPlan) Resource(org.apache.heron.spi.packing.Resource) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 43 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCompleteRamMapRequestedWithoutPaddingResource.

/**
 * Test the scenario RAM map config is completely set
 */
@Test
public void testCompleteRamMapRequestedWithoutPaddingResource() throws Exception {
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component RAM map
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.of(spoutRam), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), getDefaultUnspecifiedContainerResource(boltParallelism + spoutParallelism, numContainers, getDefaultPadding()).cloneWithRam(containerRam));
}
Also used : ByteAmount(org.apache.heron.common.basics.ByteAmount) Test(org.junit.Test)

Example 44 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCompleteRamMapRequestedWithLessThanEnoughResource.

/**
 * Test the scenario RAM map config is completely set
 */
@Test(expected = PackingException.class)
public void testCompleteRamMapRequestedWithLessThanEnoughResource() throws Exception {
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component RAM map
    ByteAmount containerRam = ByteAmount.fromGigabytes(2);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.of(spoutRam), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), getDefaultUnspecifiedContainerResource(boltParallelism + spoutParallelism, numContainers, getDefaultPadding()).cloneWithRam(containerRam));
}
Also used : ByteAmount(org.apache.heron.common.basics.ByteAmount) Test(org.junit.Test)

Example 45 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCompleteRamMapRequestedWithMoreThanEnoughResource.

/**
 * Test the scenario RAM map config is completely set
 */
@Test
public void testCompleteRamMapRequestedWithMoreThanEnoughResource() throws Exception {
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component RAM map
    ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.of(spoutRam), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), getDefaultUnspecifiedContainerResource(boltParallelism + spoutParallelism, numContainers, getDefaultPadding()).cloneWithRam(containerRam));
}
Also used : ByteAmount(org.apache.heron.common.basics.ByteAmount) Test(org.junit.Test)

Aggregations

ByteAmount (org.apache.heron.common.basics.ByteAmount)63 Test (org.junit.Test)41 HashMap (java.util.HashMap)17 Resource (org.apache.heron.spi.packing.Resource)17 PackingPlan (org.apache.heron.spi.packing.PackingPlan)14 HashSet (java.util.HashSet)11 TreeMap (java.util.TreeMap)6 Config (org.apache.heron.api.Config)6 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 EvaluatorRequest (org.apache.reef.driver.evaluator.EvaluatorRequest)2 BufferPoolMXBean (java.lang.management.BufferPoolMXBean)1 Duration (java.time.Duration)1 Map (java.util.Map)1 TopologyAPI (org.apache.heron.api.generated.TopologyAPI)1 ResourceRequirement (org.apache.heron.packing.builder.ResourceRequirement)1 BaseContainer (org.apache.heron.scheduler.mesos.framework.BaseContainer)1 InstanceId (org.apache.heron.spi.packing.InstanceId)1 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)1