use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.
the class RoundRobinPackingTest method testContainerRequestedResources.
/**
* Test the scenario container level resource config are set
*/
@Test
public void testContainerRequestedResources() throws Exception {
int numContainers = 2;
int spoutParallelism = 4;
int boltParallelism = 3;
Integer totalInstances = spoutParallelism + boltParallelism;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
// Explicit set resources for container
ByteAmount containerRam = ByteAmount.fromGigabytes(10);
ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
float containerCpu = 30;
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setContainerDiskRequested(containerDisk);
topologyConfig.setContainerCpuRequested(containerCpu);
TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
Assert.assertTrue(// due to round-off when using divide()
String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
// All instances' resource requirement should be equal
// So the size of set should be 1
Set<Resource> resources = new HashSet<>();
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
resources.add(instancePlan.getResource());
}
Assert.assertEquals(1, resources.size());
int instancesCount = containerPlan.getInstances().size();
Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), resources.iterator().next().getRam());
}
}
use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.
the class RoundRobinPackingTest method testCompleteRamMapRequested.
/**
* Test the scenario ram map config is partially set
*/
@Test
public void testCompleteRamMapRequested() throws Exception {
int numContainers = 2;
int spoutParallelism = 4;
int boltParallelism = 3;
Integer totalInstances = spoutParallelism + boltParallelism;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
// Explicit set resources for container
// the value should be ignored, since we set the complete component ram map
ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
// Explicit set component ram map
ByteAmount boltRam = ByteAmount.fromGigabytes(1);
ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setComponentRam(BOLT_NAME, boltRam);
topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}
use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.
the class CommonPackingTests method testInvalidRamInstance.
/**
* Test invalid ram for instance
*/
@Test(expected = PackingException.class)
public void testInvalidRamInstance() throws Exception {
ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
int defaultNumInstancesperContainer = 4;
// Explicit set component ram map
ByteAmount boltRam = ByteAmount.ZERO;
topologyConfig.setContainerMaxRamHint(maxContainerRam);
topologyConfig.setComponentRam(BOLT_NAME, boltRam);
TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer));
}
use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.
the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested2.
/**
* Test the scenario ram map config is fully set
*/
@Test
public void testCompleteRamMapRequested2() throws Exception {
ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
// Explicit set component ram map
ByteAmount boltRam = ByteAmount.fromGigabytes(1);
ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
topologyConfig.setContainerMaxRamHint(maxContainerRam);
topologyConfig.setComponentRam(BOLT_NAME, boltRam);
topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
Assert.assertEquals(2, packingPlanExplicitRamMap.getContainers().size());
Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, maxContainerRam);
AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}
use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.
the class TopologyUtilsTest method testGetComponentRamMapDefaultValue.
@Test
public void testGetComponentRamMapDefaultValue() {
int componentParallelism = 2;
Config topologyConfig = new Config();
Map<String, Integer> spouts = new HashMap<>();
spouts.put("spout", componentParallelism);
Map<String, Integer> bolts = new HashMap<>();
bolts.put("bolt", componentParallelism);
// sort the component ram map
Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
// Component ram map is not set, the ramMap size should be 0
Assert.assertEquals(0, ramMap.size());
}
Aggregations