Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 21 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class TopologyUtilsTest method testGetComponentRamMapSomeRamSpecified.

@Test
public void testGetComponentRamMapSomeRamSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setComponentRam("spout", spoutRam);
    // sort the component ram map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    // Component ram map sets only spout's ram
    Assert.assertArrayEquals(new String[] { "spout" }, ramMap.keySet().toArray());
    Assert.assertArrayEquals(new ByteAmount[] { spoutRam }, ramMap.values().toArray());
}

Example 22 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPacking method getContainerDiskHint.

/**
 * Provide disk per container.
 *
 * @param allocation packing output.
 * @return disk per container.
 */
private ByteAmount getContainerDiskHint(Map<Integer, List<InstanceId>> allocation) {
    ByteAmount defaultContainerDisk = instanceDiskDefault.multiply(getLargestContainerSize(allocation)).plus(DEFAULT_DISK_PADDING_PER_CONTAINER);
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    return TopologyUtils.getConfigWithDefault(topologyConfig, com.twitter.heron.api.Config.TOPOLOGY_CONTAINER_DISK_REQUESTED, defaultContainerDisk);
}

Example 23 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testRepackPadding.

/**
 * Test the scenario ram map config is partially set and scaling is requested
 */
@Test
public void testRepackPadding() throws Exception {
    int paddingPercentage = 50;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int numScalingInstances = 3;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(BOLT_NAME, numScalingInstances);
    int numContainersBeforeRepack = 3;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, boltRam, boltParallelism, instanceDefaultResources.getRam(), spoutParallelism, numContainersBeforeRepack, totalInstances);
    Assert.assertEquals(6, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (totalInstances + numScalingInstances), newPackingPlan.getInstanceCount());
    AssertPacking.assertContainers(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, instanceDefaultResources.getRam(), null);
    for (PackingPlan.ContainerPlan containerPlan : newPackingPlan.getContainers()) {
        // Each container either contains a single bolt or 1 bolt and 2 spouts or 1 bolt and 1 spout
        if (containerPlan.getInstances().size() == 1) {
            Assert.assertEquals(boltRam.increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
        if (containerPlan.getInstances().size() == 2) {
            Assert.assertEquals(boltRam.plus(instanceDefaultResources.getRam()).increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
        if (containerPlan.getInstances().size() == 3) {
            Assert.assertEquals(boltRam.plus(instanceDefaultResources.getRam().multiply(2)).increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
    }
}

Example 24 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testContainerRequestedResourcesSingleContainer.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResourcesSingleContainer() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = pack(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(Math.round(PackingUtils.increaseBy(totalInstances * instanceDefaultResources.getCpu(), DEFAULT_CONTAINER_PADDING)), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 25 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method scaleDownAndUpWithExtraPadding.

/**
 * Test the scenario where scaling down and up is simultaneously requested and padding is
 * configured
 */
@Test
public void scaleDownAndUpWithExtraPadding() throws Exception {
    int paddingPercentage = 50;
    int numContainers = 1;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
    // Explicit set component ram map
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topologyConfig.setNumStmgrs(numContainers);
    int noBolts = 2;
    int noSpouts = 1;
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(noSpouts, noBolts, topologyConfig);
    int spoutScalingUp = 1;
    int boltScalingDown = -2;
    Map<String, Integer> componentChanges = new HashMap<>();
    // 2 spouts
    componentChanges.put(SPOUT_NAME, spoutScalingUp);
    // 0 bolts
    componentChanges.put(BOLT_NAME, boltScalingDown);
    int numContainersBeforeRepack = 1;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, instanceDefaultResources.getRam(), noBolts, spoutRam, noSpouts, numContainersBeforeRepack, noSpouts + noBolts);
    Assert.assertEquals(1, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (noSpouts + noBolts + spoutScalingUp + boltScalingDown), newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, noBolts + boltScalingDown);
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, noSpouts + spoutScalingUp);
}