Examples with ByteAmount org.apache.heron.common.basics.ByteAmount used on opensource projects

Example 1 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingUtils method getComponentResourceMap.

/**
 * Compose the component resource map by reading from user configs or default
 *
 * @param components component names
 * @param componentRamMap user configured component ram map
 * @param componentCpuMap user configured component cpu map
 * @param componentDiskMap user configured component disk map
 * @param defaultInstanceResource default instance resources
 * @return component resource map
 */
public static Map<String, Resource> getComponentResourceMap(Set<String> components, Map<String, ByteAmount> componentRamMap, Map<String, Double> componentCpuMap, Map<String, ByteAmount> componentDiskMap, Resource defaultInstanceResource) {
    Map<String, Resource> componentResourceMap = new HashMap<>();
    for (String component : components) {
        ByteAmount instanceRam = componentRamMap.getOrDefault(component, defaultInstanceResource.getRam());
        double instanceCpu = componentCpuMap.getOrDefault(component, defaultInstanceResource.getCpu());
        ByteAmount instanceDisk = componentDiskMap.getOrDefault(component, defaultInstanceResource.getDisk());
        componentResourceMap.put(component, new Resource(instanceCpu, instanceRam, instanceDisk));
    }
    return componentResourceMap;
}

Example 2 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class CommonPackingTests method doPackingTestWithPartialResource.

protected PackingPlan doPackingTestWithPartialResource(TopologyAPI.Topology testTopology, Optional<ByteAmount> boltRam, Optional<Double> boltCpu, int testBoltParallelism, Optional<ByteAmount> spoutRam, Optional<Double> spoutCpu, int testSpoutParallelism, int testNumContainers, Resource padding, Resource maxContainerResource) {
    PackingPlan packingPlan = pack(testTopology);
    Assert.assertEquals(testNumContainers, packingPlan.getContainers().size());
    Assert.assertEquals(testBoltParallelism + testSpoutParallelism, (int) packingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, testBoltParallelism);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        int instancesCount = containerPlan.getInstances().size();
        if (!boltRam.isPresent() && !spoutRam.isPresent()) {
            ByteAmount instanceRam = maxContainerResource.getRam().minus(padding.getRam()).divide(instancesCount);
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceRam, instancePlan.getResource().getRam());
            }
        } else if (!boltRam.isPresent() || !spoutRam.isPresent()) {
            String explicitComponent = boltRam.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltRam.isPresent() ? SPOUT_NAME : BOLT_NAME;
            ByteAmount explicitRam = boltRam.orElseGet(spoutRam::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitRam, instancePlan.getResource().getRam());
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals(maxContainerResource.getRam().minus(explicitRam.multiply(explicitCount)).minus(padding.getRam()).divide(implicitCount), instancePlan.getResource().getRam());
                }
            }
        }
        if (!boltCpu.isPresent() && !spoutCpu.isPresent()) {
            double instanceCpu = (maxContainerResource.getCpu() - padding.getCpu()) / instancesCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceCpu, instancePlan.getResource().getCpu(), DELTA);
            }
        } else if (!boltCpu.isPresent() || !spoutCpu.isPresent()) {
            String explicitComponent = boltCpu.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltCpu.isPresent() ? SPOUT_NAME : BOLT_NAME;
            double explicitCpu = boltCpu.orElseGet(spoutCpu::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitCpu, instancePlan.getResource().getCpu(), DELTA);
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals((maxContainerResource.getCpu() - explicitCpu * explicitCount - padding.getCpu()) / implicitCount, instancePlan.getResource().getCpu(), DELTA);
                }
            }
        }
    }
    AssertPacking.assertInstanceIndices(packingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertContainerRam(packingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(packingPlan.getContainers(), maxContainerResource.getCpu());
    return packingPlan;
}

Example 3 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPackingTest method scaleDownAndUpWithExtraPadding.

/**
 * Test the scenario where scaling down and up is simultaneously requested and padding is
 * configured
 */
@Test
public void scaleDownAndUpWithExtraPadding() throws Exception {
    int paddingPercentage = 50;
    int numContainers = 2;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
    // Explicit set component RAM map
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topologyConfig.setNumStmgrs(numContainers);
    boltParallelism = 2;
    spoutParallelism = 1;
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int spoutScalingUp = 1;
    int boltScalingDown = -2;
    Map<String, Integer> componentChanges = new HashMap<>();
    // 2 spouts
    componentChanges.put(SPOUT_NAME, spoutScalingUp);
    // 0 bolts
    componentChanges.put(BOLT_NAME, boltScalingDown);
    int numContainersBeforeRepack = 2;
    int numContainersAfterRepack = 1;
    doPackingAndScalingTest(topology, componentChanges, instanceDefaultResources, boltParallelism, instanceDefaultResources.cloneWithRam(spoutRam), spoutParallelism, numContainersBeforeRepack, numContainersAfterRepack, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}

Example 4 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPackingTest method testInvalidRamInstance.

/**
 * Test invalid RAM for instance
 */
@Test(expected = PackingException.class)
public void testInvalidRamInstance() throws Exception {
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    ByteAmount boltRam = ByteAmount.ZERO;
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources, spoutParallelism, 0, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}

Example 5 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPackingTest method testInsufficientContainersWithOneAdjustment.

/**
 * Test the scenario where the user defined number of containers is not sufficient.
 */
@Test
public void testInsufficientContainersWithOneAdjustment() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.setNumStmgrs(numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, 7, getDefaultMaxContainerResource().cloneWithRam(containerRam));
}