Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 96 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPacking method packInternal.

private PackingPlan packInternal(int numContainer, Map<String, Integer> parallelismMap) {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation(numContainer, parallelismMap);
    // Get the ram map for every instance
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = getInstancesRamMapInContainer(roundRobinAllocation);
    ByteAmount containerDiskInBytes = getContainerDiskHint(roundRobinAllocation);
    double containerCpu = getContainerCpuHint(roundRobinAllocation);
    ByteAmount containerRamHint = getContainerRamHint(roundRobinAllocation);
    LOG.info(String.format("Pack internal: container cpu hint: %f, ram hint: %s, disk hint: %s.", containerCpu, containerDiskInBytes.toString(), containerRamHint.toString()));
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = containerRamPadding;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDiskDefault;
            double instanceCpu = instanceCpuDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
        }
        Resource resource = new Resource(containerCpu, containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    // Check whether it is a valid PackingPlan
    validatePackingPlan(plan);
    return plan;
}

Example 97 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class RoundRobinPacking method getInstancesRamMapInContainer.

/**
 * Calculate the ram required by any instance in the container
 *
 * @param allocation the allocation of instances in different container
 * @return A map: (containerId -> (instanceId -> instanceRequiredRam))
 */
private Map<Integer, Map<InstanceId, ByteAmount>> getInstancesRamMapInContainer(Map<Integer, List<InstanceId>> allocation) {
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMapInContainer = new HashMap<>();
    ByteAmount containerRamHint = getContainerRamHint(allocation);
    for (int containerId : allocation.keySet()) {
        List<InstanceId> instanceIds = allocation.get(containerId);
        Map<InstanceId, ByteAmount> ramInsideContainer = new HashMap<>();
        instancesRamMapInContainer.put(containerId, ramInsideContainer);
        List<InstanceId> instancesToBeAccounted = new ArrayList<>();
        // Calculate the actual value
        ByteAmount usedRam = ByteAmount.ZERO;
        for (InstanceId instanceId : instanceIds) {
            String componentName = instanceId.getComponentName();
            if (ramMap.containsKey(componentName)) {
                ByteAmount ram = ramMap.get(componentName);
                ramInsideContainer.put(instanceId, ram);
                usedRam = usedRam.plus(ram);
            } else {
                instancesToBeAccounted.add(instanceId);
            }
        }
        // Now we have calculated ram for instances specified in ComponentRamMap
        // Then to calculate ram for the rest instances
        int instancesToAllocate = instancesToBeAccounted.size();
        if (instancesToAllocate != 0) {
            ByteAmount individualInstanceRam = instanceRamDefault;
            // If container ram is specified
            if (!containerRamHint.equals(NOT_SPECIFIED_NUMBER_VALUE)) {
                // remove ram for heron internal process
                ByteAmount remainingRam = containerRamHint.minus(containerRamPadding).minus(usedRam);
                // Split remaining ram evenly
                individualInstanceRam = remainingRam.divide(instancesToAllocate);
            }
            // Put the results in instancesRam
            for (InstanceId instanceId : instancesToBeAccounted) {
                ramInsideContainer.put(instanceId, individualInstanceRam);
            }
        }
    }
    return instancesRamMapInContainer;
}

Example 98 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPacking method newPackingPlanBuilder.

private PackingPlanBuilder newPackingPlanBuilder(PackingPlan existingPackingPlan) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    int paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    Resource maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_REQUESTED, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_REQUESTED, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_DISK_REQUESTED, defaultDisk.increaseBy(paddingPercentage)));
    LOG.info(String.format("ResourceCompliantRRPacking newPackingPlanBuilder. " + "CPU max: %f, RAMmaxMax: %s, DISK max: %s, Padding percentage: %d.", maxContainerResources.getCpu(), maxContainerResources.getRam().toString(), maxContainerResources.getDisk().toString(), paddingPercentage));
    return new PackingPlanBuilder(topology.getId(), existingPackingPlan).setMaxContainerResource(maxContainerResources).setDefaultInstanceResource(defaultInstanceResources).setRequestedContainerPadding(paddingPercentage).setRequestedComponentRam(TopologyUtils.getComponentRamMapConfig(topology));
}

Example 99 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testInsufficientContainersWithOneAdjustment.

/**
 * Test the scenario where the user defined number of containers is not sufficient.
 */
@Test
public void testInsufficientContainersWithOneAdjustment() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    TopologyAPI.Topology newTopology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = pack(newTopology);
    Assert.assertEquals(7, packingPlan.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlan.getInstanceCount());
}

Example 100 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ResourceCompliantRRPackingTest method testPartialRamMapScaling.

/**
 * Test the scenario ram map config is partially set and scaling is requested
 */
@Test
public void testPartialRamMapScaling() throws Exception {
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int numScalingInstances = 3;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(BOLT_NAME, numScalingInstances);
    int numContainersBeforeRepack = 3;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, boltRam, boltParallelism, instanceDefaultResources.getRam(), spoutParallelism, numContainersBeforeRepack, totalInstances);
    Assert.assertEquals(6, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (totalInstances + numScalingInstances), newPackingPlan.getInstanceCount());
    AssertPacking.assertContainers(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, instanceDefaultResources.getRam(), null);
}