Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 6 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingPlanBuilder method buildContainerPlans.

/**
   * Estimate the per instance and topology resources for the packing plan based on the ramMap,
   * instance defaults and paddingPercentage.
   *
   * @return container plans
   */
private static Set<PackingPlan.ContainerPlan> buildContainerPlans(Map<Integer, Container> containerInstances, Map<String, ByteAmount> ramMap, Resource instanceDefaults, int paddingPercentage) {
    Set<PackingPlan.ContainerPlan> containerPlans = new LinkedHashSet<>();
    for (Integer containerId : containerInstances.keySet()) {
        Container container = containerInstances.get(containerId);
        if (container.getInstances().size() == 0) {
            continue;
        }
        ByteAmount containerRam = ByteAmount.ZERO;
        ByteAmount containerDiskInBytes = ByteAmount.ZERO;
        double containerCpu = 0;
        // Calculate the resource required for single instance
        Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : container.getInstances()) {
            InstanceId instanceId = new InstanceId(instancePlan.getComponentName(), instancePlan.getTaskId(), instancePlan.getComponentIndex());
            ByteAmount instanceRam;
            if (ramMap.containsKey(instanceId.getComponentName())) {
                instanceRam = ramMap.get(instanceId.getComponentName());
            } else {
                instanceRam = instanceDefaults.getRam();
            }
            containerRam = containerRam.plus(instanceRam);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDefaults.getDisk();
            containerDiskInBytes = containerDiskInBytes.plus(instanceDisk);
            double instanceCpu = instanceDefaults.getCpu();
            containerCpu += instanceCpu;
            // Insert it into the map
            instancePlans.add(new PackingPlan.InstancePlan(instanceId, new Resource(instanceCpu, instanceRam, instanceDisk)));
        }
        containerCpu += (paddingPercentage * containerCpu) / 100;
        containerRam = containerRam.increaseBy(paddingPercentage);
        containerDiskInBytes = containerDiskInBytes.increaseBy(paddingPercentage);
        Resource resource = new Resource(Math.round(containerCpu), containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
        containerPlans.add(containerPlan);
    }
    return containerPlans;
}

Example 7 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPacking method getInstancesRamMapInContainer.

/**
   * Calculate the ram required by any instance in the container
   *
   * @param allocation the allocation of instances in different container
   * @return A map: (containerId -> (instanceId -> instanceRequiredRam))
   */
private Map<Integer, Map<InstanceId, ByteAmount>> getInstancesRamMapInContainer(Map<Integer, List<InstanceId>> allocation) {
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMapInContainer = new HashMap<>();
    for (int containerId : allocation.keySet()) {
        List<InstanceId> instanceIds = allocation.get(containerId);
        Map<InstanceId, ByteAmount> ramInsideContainer = new HashMap<>();
        instancesRamMapInContainer.put(containerId, ramInsideContainer);
        List<InstanceId> instancesToBeAccounted = new ArrayList<>();
        // Calculate the actual value
        ByteAmount usedRam = ByteAmount.ZERO;
        for (InstanceId instanceId : instanceIds) {
            String componentName = instanceId.getComponentName();
            if (ramMap.containsKey(componentName)) {
                ByteAmount ram = ramMap.get(componentName);
                ramInsideContainer.put(instanceId, ram);
                usedRam = usedRam.plus(ram);
            } else {
                instancesToBeAccounted.add(instanceId);
            }
        }
        // Now we have calculated ram for instances specified in ComponentRamMap
        // Then to calculate ram for the rest instances
        ByteAmount containerRamHint = getContainerRamHint(allocation);
        int instancesToAllocate = instancesToBeAccounted.size();
        if (instancesToAllocate != 0) {
            ByteAmount individualInstanceRam = instanceRamDefault;
            // If container ram is specified
            if (!containerRamHint.equals(NOT_SPECIFIED_NUMBER_VALUE)) {
                // remove ram for heron internal process
                ByteAmount remainingRam = containerRamHint.minus(DEFAULT_RAM_PADDING_PER_CONTAINER).minus(usedRam);
                // Split remaining ram evenly
                individualInstanceRam = remainingRam.divide(instancesToAllocate);
            }
            // Put the results in instancesRam
            for (InstanceId instanceId : instancesToBeAccounted) {
                ramInsideContainer.put(instanceId, individualInstanceRam);
            }
        }
    }
    return instancesRamMapInContainer;
}

Example 8 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPacking method getContainerDiskHint.

/**
   * Provide disk per container.
   *
   * @param allocation packing output.
   * @return disk per container.
   */
private ByteAmount getContainerDiskHint(Map<Integer, List<InstanceId>> allocation) {
    ByteAmount defaultContainerDisk = instanceDiskDefault.multiply(getLargestContainerSize(allocation)).plus(DEFAULT_DISK_PADDING_PER_CONTAINER);
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    return TopologyUtils.getConfigWithDefault(topologyConfig, com.twitter.heron.api.Config.TOPOLOGY_CONTAINER_DISK_REQUESTED, defaultContainerDisk);
}

Example 9 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingUtils method getResourceRequirement.

public static Resource getResourceRequirement(String component, Map<String, ByteAmount> componentRamMap, Resource defaultInstanceResource, Resource maxContainerResource, int paddingPercentage) {
    ByteAmount instanceRam = defaultInstanceResource.getRam();
    if (componentRamMap.containsKey(component)) {
        instanceRam = componentRamMap.get(component);
    }
    assertIsValidInstance(defaultInstanceResource.cloneWithRam(instanceRam), MIN_RAM_PER_INSTANCE, maxContainerResource, paddingPercentage);
    return defaultInstanceResource.cloneWithRam(instanceRam);
}

Example 10 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPackingTest method testRepackPadding.

/**
   * Test the scenario ram map config is partially set and scaling is requested
   */
@Test
public void testRepackPadding() throws Exception {
    int paddingPercentage = 50;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    int numScalingInstances = 3;
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(BOLT_NAME, numScalingInstances);
    int numContainersBeforeRepack = 3;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, boltRam, boltParallelism, instanceDefaultResources.getRam(), spoutParallelism, numContainersBeforeRepack, totalInstances);
    Assert.assertEquals(6, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (totalInstances + numScalingInstances), newPackingPlan.getInstanceCount());
    AssertPacking.assertContainers(newPackingPlan.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, instanceDefaultResources.getRam(), null);
    for (PackingPlan.ContainerPlan containerPlan : newPackingPlan.getContainers()) {
        //Each container either contains a single bolt or 1 bolt and 2 spouts or 1 bolt and 1 spout
        if (containerPlan.getInstances().size() == 1) {
            Assert.assertEquals(boltRam.increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
        if (containerPlan.getInstances().size() == 2) {
            Assert.assertEquals(boltRam.plus(instanceDefaultResources.getRam()).increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
        if (containerPlan.getInstances().size() == 3) {
            Assert.assertEquals(boltRam.plus(instanceDefaultResources.getRam().multiply(2)).increaseBy(paddingPercentage), containerPlan.getRequiredResource().getRam());
        }
    }
}