Examples with ByteAmount org.apache.heron.common.basics.ByteAmount used on opensource projects

Example 36 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPacking method packInternal.

private PackingPlan packInternal(int numContainer, Map<String, Integer> parallelismMap) {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation(numContainer, parallelismMap);
    Resource containerResourceHint = getContainerResourceHint(roundRobinAllocation);
    int largestContainerSize = getLargestContainerSize(roundRobinAllocation);
    // Get the RAM map for every instance
    ByteAmount containerRamDefault = instanceRamDefault.multiply(largestContainerSize).plus(containerRamPadding);
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = calculateInstancesResourceMapInContainer(roundRobinAllocation, TopologyUtils.getComponentRamMapConfig(topology), containerResourceHint.getRam(), containerRamDefault, instanceRamDefault, containerRamPadding, ByteAmount.ZERO, NOT_SPECIFIED_BYTE_AMOUNT, RAM);
    // Get the CPU map for every instance
    float containerCPUDefault = Math.round(instanceCpuDefault * largestContainerSize + containerCpuPadding);
    Map<Integer, Map<InstanceId, CPUShare>> instancesCpuMap = calculateInstancesResourceMapInContainer(roundRobinAllocation, CPUShare.convertDoubleMapToCpuShareMap(TopologyUtils.getComponentCpuMapConfig(topology)), CPUShare.fromDouble(containerResourceHint.getCpu()), CPUShare.fromDouble(containerCPUDefault), CPUShare.fromDouble(instanceCpuDefault), CPUShare.fromDouble(containerCpuPadding), CPUShare.fromDouble(0.0), CPUShare.fromDouble(NOT_SPECIFIED_CPU_SHARE), CPU);
    LOG.info(String.format("Pack internal: container CPU hint: %.3f, RAM hint: %s, disk hint: %s.", containerResourceHint.getCpu(), containerResourceHint.getRam().toString(), containerResourceHint.getDisk().toString()));
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = ByteAmount.ZERO;
        double containerCpu = 0.0;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            Double instanceCpu = instancesCpuMap.get(containerId).get(instanceId).getValue();
            // Currently not yet support disk config for different components, just use the default.
            ByteAmount instanceDisk = instanceDiskDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
            containerCpu += instanceCpu;
        }
        // finalize container resource
        containerCpu += containerCpuPadding;
        if (containerResourceHint.getCpu() != NOT_SPECIFIED_CPU_SHARE) {
            containerCpu = Math.min(containerCpu, containerResourceHint.getCpu());
        }
        containerRam = containerRam.plus(containerRamPadding);
        if (!containerResourceHint.getRam().equals(NOT_SPECIFIED_BYTE_AMOUNT)) {
            containerRam = ByteAmount.fromBytes(Math.min(containerRam.asBytes(), containerResourceHint.getRam().asBytes()));
        }
        ByteAmount containerDisk = containerResourceHint.getDisk();
        if (containerDisk.equals(NOT_SPECIFIED_BYTE_AMOUNT)) {
            containerDisk = instanceDiskDefault.multiply(largestContainerSize).plus(DEFAULT_DISK_PADDING_PER_CONTAINER);
        }
        Resource resource = new Resource(Math.max(containerCpu, containerResourceHint.getCpu()), containerRam, containerDisk);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
        LOG.info(String.format("Pack internal finalized: container#%d CPU: %f, RAM: %s, disk: %s.", containerId, resource.getCpu(), resource.getRam().toString(), resource.getDisk().toString()));
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    validatePackingPlan(plan);
    return plan;
}

Example 37 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPacking method getSortedRAMComponents.

/**
 * Sort the components in decreasing order based on their RAM requirements
 *
 * @return The sorted list of components and their RAM requirements
 */
private ArrayList<ResourceRequirement> getSortedRAMComponents(Set<String> componentNames) {
    ArrayList<ResourceRequirement> resourceRequirements = new ArrayList<>();
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    for (String componentName : componentNames) {
        resourceRequirements.add(new ResourceRequirement(componentName, ramMap.getOrDefault(componentName, ByteAmount.ZERO)));
    }
    Collections.sort(resourceRequirements, SortingStrategy.RAM_FIRST.reversed());
    return resourceRequirements;
}

Example 38 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class AbstractPacking method setPackingConfigs.

/**
 * Instatiate the packing algorithm parameters related to this topology.
 */
private void setPackingConfigs(Config config) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    // instance default resources are acquired from heron system level config
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    int paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    ByteAmount ramPadding = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING);
    double cpuPadding = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_CPU_PADDING);
    Resource preliminaryPadding = new Resource(cpuPadding, ramPadding, PackingUtils.DEFAULT_CONTAINER_DISK_PADDING);
    this.maxNumInstancesPerContainer = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_NUM_INSTANCES, PackingUtils.DEFAULT_MAX_NUM_INSTANCES_PER_CONTAINER);
    // container default resources are computed as:
    // max number of instances per container * default instance resources
    double containerDefaultCpu = this.defaultInstanceResources.getCpu() * maxNumInstancesPerContainer;
    ByteAmount containerDefaultRam = this.defaultInstanceResources.getRam().multiply(maxNumInstancesPerContainer);
    ByteAmount containerDefaultDisk = this.defaultInstanceResources.getDisk().multiply(maxNumInstancesPerContainer);
    double containerCpu = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_REQUESTED, containerDefaultCpu);
    ByteAmount containerRam = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_REQUESTED, containerDefaultRam);
    ByteAmount containerDisk = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_DISK_REQUESTED, containerDefaultDisk);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // finalize padding
    this.padding = PackingUtils.finalizePadding(containerResource, preliminaryPadding, paddingPercentage);
    // finalize container resources
    this.maxContainerResources = containerResource;
    this.componentResourceMap = PackingUtils.getComponentResourceMap(TopologyUtils.getComponentParallelism(topology).keySet(), TopologyUtils.getComponentRamMapConfig(topology), TopologyUtils.getComponentCpuMapConfig(topology), TopologyUtils.getComponentDiskMapConfig(topology), defaultInstanceResources);
}

Example 39 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCheckInsufficientRamFailure.

@Test(expected = PackingException.class)
public void testCheckInsufficientRamFailure() throws Exception {
    // Explicit set insufficient RAM for container
    ByteAmount containerRam = ByteAmount.ZERO;
    topologyConfig.setContainerRamRequested(containerRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, numContainers, getDefaultUnspecifiedContainerResource(boltParallelism + spoutParallelism, numContainers, getDefaultPadding()).cloneWithRam(containerRam));
}

Example 40 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testPartialRamMapWithoutContainerRequestedResources.

@Test
public void testPartialRamMapWithoutContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    // max container resource is 6G
    ByteAmount containerRam = ByteAmount.fromGigabytes(6);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    // Don't set container RAM
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.of(boltRam), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        int instancesCount = containerPlan.getInstances().size();
        if (instancesCount == 4) {
            // Biggest container
            Assert.assertEquals(1, differentResources.size());
        } else {
            // Smaller container
            Assert.assertEquals(2, differentResources.size());
        }
    }
}