Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 41 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested.

/**
 * Test the scenario ram map config is fully set
 */
@Test
public void testCompleteRamMapRequested() throws Exception {
    // Explicit set max resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(15);
    ByteAmount maxContainerDisk = ByteAmount.fromGigabytes(20);
    double maxContainerCpu = 30;
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setContainerMaxDiskHint(maxContainerDisk);
    topologyConfig.setContainerMaxCpuHint(maxContainerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(1, packingPlanExplicitRamMap.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, maxContainerRam);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}

Example 42 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPackingTest method testContainerRequestedResourcesWhenRamPaddingSet.

/**
 * Test the scenario container level resource config are set
 */
@Test
public void testContainerRequestedResourcesWhenRamPaddingSet() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    ByteAmount containerRamPadding = ByteAmount.fromMegabytes(512);
    double containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setContainerRamPadding(containerRamPadding);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
        Assert.assertTrue(// due to round-off when using divide()
        String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
        Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(containerRamPadding).divide(instancesCount), resources.iterator().next().getRam());
    }
}

Example 43 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPackingTest method testCheckFailure.

@Test(expected = PackingException.class)
public void testCheckFailure() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set insufficient ram for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(0);
    topologyConfig.setContainerRamRequested(containerRam);
    TopologyAPI.Topology topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    getRoundRobinPackingPlan(topology);
}

Example 44 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPackingTest method testCompleteRamMapRequested.

/**
 * Test the scenario ram map config is partially set
 */
@Test
public void testCompleteRamMapRequested() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}

Example 45 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class PackingPlan method getMaxContainerResources.

/**
 * Computes the maximum of all the resources required by the containers in the packing plan. If
 * the PackingPlan has already been scheduled, the scheduled resources will be used over the
 * required resources.
 *
 * @return maximum Resources found in all containers.
 */
public Resource getMaxContainerResources() {
    double maxCpu = 0;
    ByteAmount maxRam = ByteAmount.ZERO;
    ByteAmount maxDisk = ByteAmount.ZERO;
    for (ContainerPlan containerPlan : getContainers()) {
        Resource containerResource = containerPlan.getScheduledResource().or(containerPlan.getRequiredResource());
        maxCpu = Math.max(maxCpu, containerResource.getCpu());
        maxRam = maxRam.max(containerResource.getRam());
        maxDisk = maxDisk.max(containerResource.getDisk());
    }
    return new Resource(maxCpu, maxRam, maxDisk);
}