Examples with ByteAmount org.apache.heron.common.basics.ByteAmount used on opensource projects

Example 56 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingUtils method finalizePadding.

/**
 * Finalize padding by taking Math.max(containerResource * paddingPercent, paddingValue)
 *
 * @param containerResource max container resource
 * @param padding padding value
 * @param paddingPercentage padding percent
 * @return finalized padding amount
 */
public static Resource finalizePadding(Resource containerResource, Resource padding, int paddingPercentage) {
    double cpuPadding = Math.max(padding.getCpu(), containerResource.getCpu() * paddingPercentage / 100);
    ByteAmount ramPadding = ByteAmount.fromBytes(Math.max(padding.getRam().asBytes(), containerResource.getRam().asBytes() * paddingPercentage / 100));
    ByteAmount diskPadding = ByteAmount.fromBytes(Math.max(padding.getDisk().asBytes(), containerResource.getDisk().asBytes() * paddingPercentage / 100));
    return new Resource(cpuPadding, ramPadding, diskPadding);
}

Example 57 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class CommonPackingTests method doScalingTestWithPartialResource.

protected PackingPlan doScalingTestWithPartialResource(TopologyAPI.Topology testTopology, PackingPlan packingPlan, Map<String, Integer> componentChanges, Optional<ByteAmount> boltRam, Optional<Double> boltCpu, int testBoltParallelism, Optional<ByteAmount> spoutRam, Optional<Double> spoutCpu, int testSpoutParallelism, int testNumContainers, Resource padding, Resource maxContainerResource) {
    System.out.println(packingPlan);
    PackingPlan newPackingPlan = repack(testTopology, packingPlan, componentChanges);
    System.out.println(newPackingPlan);
    Assert.assertEquals(testNumContainers, newPackingPlan.getContainers().size());
    Assert.assertEquals(testBoltParallelism + testSpoutParallelism + componentChanges.getOrDefault(BOLT_NAME, 0) + componentChanges.getOrDefault(SPOUT_NAME, 0), (int) newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, testBoltParallelism + componentChanges.getOrDefault(BOLT_NAME, 0));
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, testSpoutParallelism + componentChanges.getOrDefault(SPOUT_NAME, 0));
    for (PackingPlan.ContainerPlan containerPlan : newPackingPlan.getContainers()) {
        int instancesCount = containerPlan.getInstances().size();
        if (!boltRam.isPresent() && !spoutRam.isPresent()) {
            ByteAmount instanceRam = maxContainerResource.getRam().minus(padding.getRam()).divide(instancesCount);
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceRam, instancePlan.getResource().getRam());
            }
        } else if (!boltRam.isPresent() || !spoutRam.isPresent()) {
            String explicitComponent = boltRam.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltRam.isPresent() ? SPOUT_NAME : BOLT_NAME;
            ByteAmount explicitRam = boltRam.orElseGet(spoutRam::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitRam, instancePlan.getResource().getRam());
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals(maxContainerResource.getRam().minus(explicitRam.multiply(explicitCount)).minus(padding.getRam()).divide(implicitCount), instancePlan.getResource().getRam());
                }
            }
        }
        if (!boltCpu.isPresent() && !spoutCpu.isPresent()) {
            double instanceCpu = (maxContainerResource.getCpu() - padding.getCpu()) / instancesCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                Assert.assertEquals(instanceCpu, instancePlan.getResource().getCpu(), DELTA);
            }
        } else if (!boltCpu.isPresent() || !spoutCpu.isPresent()) {
            String explicitComponent = boltCpu.isPresent() ? BOLT_NAME : SPOUT_NAME;
            String implicitComponent = boltCpu.isPresent() ? SPOUT_NAME : BOLT_NAME;
            double explicitCpu = boltCpu.orElseGet(spoutCpu::get);
            int explicitCount = 0;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(explicitComponent)) {
                    Assert.assertEquals(explicitCpu, instancePlan.getResource().getCpu(), DELTA);
                    explicitCount++;
                }
            }
            int implicitCount = instancesCount - explicitCount;
            for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
                if (instancePlan.getComponentName().equals(implicitComponent)) {
                    Assert.assertEquals((maxContainerResource.getCpu() - explicitCpu * explicitCount - padding.getCpu()) / implicitCount, instancePlan.getResource().getCpu(), DELTA);
                }
            }
        }
    }
    AssertPacking.assertInstanceIndices(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME);
    AssertPacking.assertContainerRam(newPackingPlan.getContainers(), maxContainerResource.getRam());
    AssertPacking.assertContainerCpu(newPackingPlan.getContainers(), maxContainerResource.getCpu());
    return newPackingPlan;
}

Example 58 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testPartialRamMapScaling.

/**
 * Test the scenario RAM map config is partially set and scaling is requested
 */
@Test
public void testPartialRamMapScaling() throws Exception {
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int numScalingInstances = 3;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(BOLT_NAME, numScalingInstances);
    int numContainersBeforeRepack = 3;
    int numContainersAfterRepack = 4;
    doPackingAndScalingTest(topology, componentChanges, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources, spoutParallelism, numContainersBeforeRepack, numContainersAfterRepack, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}

Example 59 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testPartialRamMapWithPadding.

/**
 * Test the scenario RAM map config is partially set and padding is configured
 */
@Test
public void testPartialRamMapWithPadding() throws Exception {
    topologyConfig.setContainerPaddingPercentage(0);
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources, spoutParallelism, 3, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}

Example 60 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testRepackPadding.

/**
 * Test the scenario RAM map config is partially set and scaling is requested
 */
@Test
public void testRepackPadding() throws Exception {
    int paddingPercentage = 50;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set component RAM map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int numScalingInstances = 3;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(BOLT_NAME, numScalingInstances);
    int numContainersBeforeRepack = 4;
    int numContainersAfterRepack = 6;
    doPackingAndScalingTest(topology, componentChanges, instanceDefaultResources.cloneWithRam(boltRam), boltParallelism, instanceDefaultResources, spoutParallelism, numContainersBeforeRepack, numContainersAfterRepack, getDefaultMaxContainerResource().cloneWithRam(maxContainerRam));
}