Search in sources :

Example 16 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testContainerRequestedResources.

/**
   * Test the scenario container level resource config are set
   */
@Test
public void testContainerRequestedResources() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
        Assert.assertTrue(// due to round-off when using divide()
        String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
        Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), resources.iterator().next().getRam());
    }
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) Config(com.twitter.heron.spi.common.Config) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) Resource(com.twitter.heron.spi.packing.Resource) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 17 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCompleteRamMapRequested.

/**
   * Test the scenario ram map config is partially set
   */
@Test
public void testCompleteRamMapRequested() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) Config(com.twitter.heron.spi.common.Config) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) Test(org.junit.Test)

Example 18 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class CommonPackingTests method testInvalidRamInstance.

/**
   * Test invalid ram for instance
   */
@Test(expected = PackingException.class)
public void testInvalidRamInstance() throws Exception {
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    int defaultNumInstancesperContainer = 4;
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.ZERO;
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer));
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) Test(org.junit.Test)

Example 19 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested2.

/**
   * Test the scenario ram map config is fully set
   */
@Test
public void testCompleteRamMapRequested2() throws Exception {
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(2, packingPlanExplicitRamMap.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, maxContainerRam);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) PackingPlan(com.twitter.heron.spi.packing.PackingPlan) TopologyAPI(com.twitter.heron.api.generated.TopologyAPI) Test(org.junit.Test)

Example 20 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class TopologyUtilsTest method testGetComponentRamMapDefaultValue.

@Test
public void testGetComponentRamMapDefaultValue() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    // sort the component ram map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    // Component ram map is not set, the ramMap size should be 0
    Assert.assertEquals(0, ramMap.size());
}
Also used : ByteAmount(com.twitter.heron.common.basics.ByteAmount) HashMap(java.util.HashMap) Config(com.twitter.heron.api.Config) TreeMap(java.util.TreeMap) Test(org.junit.Test)

Aggregations

ByteAmount (com.twitter.heron.common.basics.ByteAmount)109 Test (org.junit.Test)64 PackingPlan (com.twitter.heron.spi.packing.PackingPlan)61 TopologyAPI (com.twitter.heron.api.generated.TopologyAPI)55 HashMap (java.util.HashMap)30 Resource (com.twitter.heron.spi.packing.Resource)29 HashSet (java.util.HashSet)15 Config (com.twitter.heron.api.Config)9 Config (com.twitter.heron.spi.common.Config)9 TreeMap (java.util.TreeMap)9 InstanceId (com.twitter.heron.spi.packing.InstanceId)6 ArrayList (java.util.ArrayList)6 VisibleForTesting (com.google.common.annotations.VisibleForTesting)4 Map (java.util.Map)4 EvaluatorRequest (org.apache.reef.driver.evaluator.EvaluatorRequest)4 List (java.util.List)3 RamRequirement (com.twitter.heron.packing.RamRequirement)2 ResourceExceededException (com.twitter.heron.packing.ResourceExceededException)2 PackingPlanBuilder (com.twitter.heron.packing.builder.PackingPlanBuilder)2 BaseContainer (com.twitter.heron.scheduler.mesos.framework.BaseContainer)2