Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 26 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testInsufficientContainersWithMultipleAdjustments.

/**
 * Test the scenario where the user defined number of containers is not sufficient.
 */
@Test
public void testInsufficientContainersWithMultipleAdjustments() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(3);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = pack(topologyExplicitRamMap);
    Assert.assertEquals(7, packingPlan.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlan.getInstanceCount());
}

Example 27 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testPartialRamMap.

/**
 * Test the scenario ram map config is fully set
 */
@Test
public void testPartialRamMap() throws Exception {
    int numContainers = 2;
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    Assert.assertEquals(numContainers, packingPlanExplicitRamMap.getContainers().size());
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
}

Example 28 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class PackingUtils method assertIsValidInstance.

/**
 * Verifies the Instance has enough RAM and that it can fit within the container limits.
 *
 * @param instanceResources The resources allocated to the instance
 * @throws PackingException if the instance is invalid
 */
private static void assertIsValidInstance(Resource instanceResources, ByteAmount minInstanceRam, Resource maxContainerResources, int paddingPercentage) throws PackingException {
    if (instanceResources.getRam().lessThan(minInstanceRam)) {
        throw new PackingException(String.format("Instance requires ram %s which is less than the minimum ram per instance of %s", instanceResources.getRam(), minInstanceRam));
    }
    ByteAmount instanceRam = instanceResources.getRam().increaseBy(paddingPercentage);
    if (instanceRam.greaterThan(maxContainerResources.getRam())) {
        throw new PackingException(String.format("This instance requires containers of at least %s ram. The current max container " + "size is %s", instanceRam, maxContainerResources.getRam()));
    }
    double instanceCpu = Math.round(PackingUtils.increaseBy(instanceResources.getCpu(), paddingPercentage));
    if (instanceCpu > maxContainerResources.getCpu()) {
        throw new PackingException(String.format("This instance requires containers with at least %s cpu cores. The current max container" + "size is %s cores", instanceCpu, maxContainerResources.getCpu()));
    }
    ByteAmount instanceDisk = instanceResources.getDisk().increaseBy(paddingPercentage);
    if (instanceDisk.greaterThan(maxContainerResources.getDisk())) {
        throw new PackingException(String.format("This instance requires containers of at least %s disk. The current max container" + "size is %s", instanceDisk, maxContainerResources.getDisk()));
    }
}

Example 29 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class CommonPackingTests method testInvalidRamInstance.

/**
 * Test invalid ram for instance
 */
@Test(expected = PackingException.class)
public void testInvalidRamInstance() throws Exception {
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    int defaultNumInstancesperContainer = 4;
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.ZERO;
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer));
}

Example 30 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class Container method assertHasSpace.

/**
 * Check whether the container can accommodate a new instance with specific resource requirements
 */
private void assertHasSpace(Resource resource) throws ResourceExceededException {
    Resource usedResources = this.getTotalUsedResources();
    ByteAmount newRam = usedResources.getRam().plus(resource.getRam()).increaseBy(paddingPercentage);
    double newCpu = Math.round(PackingUtils.increaseBy(usedResources.getCpu() + resource.getCpu(), paddingPercentage));
    ByteAmount newDisk = usedResources.getDisk().plus(resource.getDisk()).increaseBy(paddingPercentage);
    if (newRam.greaterThan(this.capacity.getRam())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of ram to existing %s " + "bytes with %d percent padding would exceed capacity %s", resource.getRam(), usedResources.getRam(), paddingPercentage, this.capacity.getRam()));
    }
    if (newCpu > this.capacity.getCpu()) {
        throw new ResourceExceededException(String.format("Adding %s cores to existing %s " + "cores with %d percent padding would exceed capacity %s", resource.getCpu(), usedResources.getCpu(), paddingPercentage, this.capacity.getCpu()));
    }
    if (newDisk.greaterThan(this.capacity.getDisk())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of disk to existing %s " + "bytes with %s percent padding would exceed capacity %s", resource.getDisk(), usedResources.getDisk(), paddingPercentage, this.capacity.getDisk()));
    }
}