Examples with Resource com.twitter.heron.spi.packing.Resource used on opensource projects

Example 6 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method buildContainerPlans.

/**
   * Estimate the per instance and topology resources for the packing plan based on the ramMap,
   * instance defaults and paddingPercentage.
   *
   * @return container plans
   */
private static Set<PackingPlan.ContainerPlan> buildContainerPlans(Map<Integer, Container> containerInstances, Map<String, ByteAmount> ramMap, Resource instanceDefaults, int paddingPercentage) {
    Set<PackingPlan.ContainerPlan> containerPlans = new LinkedHashSet<>();
    for (Integer containerId : containerInstances.keySet()) {
        Container container = containerInstances.get(containerId);
        if (container.getInstances().size() == 0) {
            continue;
        }
        ByteAmount containerRam = ByteAmount.ZERO;
        ByteAmount containerDiskInBytes = ByteAmount.ZERO;
        double containerCpu = 0;
        // Calculate the resource required for single instance
        Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : container.getInstances()) {
            InstanceId instanceId = new InstanceId(instancePlan.getComponentName(), instancePlan.getTaskId(), instancePlan.getComponentIndex());
            ByteAmount instanceRam;
            if (ramMap.containsKey(instanceId.getComponentName())) {
                instanceRam = ramMap.get(instanceId.getComponentName());
            } else {
                instanceRam = instanceDefaults.getRam();
            }
            containerRam = containerRam.plus(instanceRam);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDefaults.getDisk();
            containerDiskInBytes = containerDiskInBytes.plus(instanceDisk);
            double instanceCpu = instanceDefaults.getCpu();
            containerCpu += instanceCpu;
            // Insert it into the map
            instancePlans.add(new PackingPlan.InstancePlan(instanceId, new Resource(instanceCpu, instanceRam, instanceDisk)));
        }
        containerCpu += (paddingPercentage * containerCpu) / 100;
        containerRam = containerRam.increaseBy(paddingPercentage);
        containerDiskInBytes = containerDiskInBytes.increaseBy(paddingPercentage);
        Resource resource = new Resource(Math.round(containerCpu), containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
        containerPlans.add(containerPlan);
    }
    return containerPlans;
}

Example 7 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPackingTest method testContainerRequestedResourcesSingleContainer.

/**
   * Test the scenario where container level resource config are set
   */
@Test
public void testContainerRequestedResourcesSingleContainer() throws Exception {
    int numContainers = 1;
    // Set up the topology and its config
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = pack(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(Math.round(PackingUtils.increaseBy(totalInstances * instanceDefaultResources.getCpu(), DEFAULT_CONTAINER_PADDING)), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 8 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testContainerRequestedResources.

/**
   * Test the scenario container level resource config are set
   */
@Test
public void testContainerRequestedResources() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
        Assert.assertTrue(// due to round-off when using divide()
        String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
        Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), resources.iterator().next().getRam());
    }
}

Example 9 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingTestHelper method generateTestPackingPlan.

/**
   * Returns a PackingPlan to use for testing scale up/down that has instances of specific
   * components on specific containers.
   */
private static PackingPlan generateTestPackingPlan(String topologyName, PackingPlan previousPackingPlan, Pair<Integer, String>[] addInstances, Pair<Integer, String>[] removeInstances, int containerPadding) throws ResourceExceededException {
    PackingPlanBuilder builder = new PackingPlanBuilder(topologyName, previousPackingPlan);
    int instanceCount = 0;
    if (previousPackingPlan != null) {
        instanceCount = previousPackingPlan.getInstanceCount();
    } else if (addInstances != null) {
        instanceCount = addInstances.length;
    }
    // use basic default resource to allow all instances to fit on a single container, if that's
    // what the tester desired. We can extend this to permit passing custom resource requirements
    // as needed.
    builder.setDefaultInstanceResource(new Resource(1, ByteAmount.fromMegabytes(192), ByteAmount.fromMegabytes(1)));
    builder.setMaxContainerResource(new Resource(instanceCount, ByteAmount.fromMegabytes(192).multiply(instanceCount), ByteAmount.fromMegabytes(instanceCount)));
    // This setting is important, see https://github.com/twitter/heron/issues/1577
    builder.setRequestedContainerPadding(containerPadding);
    if (addInstances != null) {
        for (Pair<Integer, String> componentInstance : addInstances) {
            builder.addInstance(componentInstance.first, componentInstance.second);
        }
    }
    if (removeInstances != null) {
        for (Pair<Integer, String> componentInstance : removeInstances) {
            builder.removeInstance(componentInstance.first, componentInstance.second);
        }
    }
    return builder.build();
}

Example 10 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class ScorerTest method init.

@Before
public void init() throws ResourceExceededException {
    Resource containerCapacity = new Resource(1000, ByteAmount.fromGigabytes(100), ByteAmount.fromGigabytes(100));
    testContainers = new Container[] { new Container(1, containerCapacity, 0), new Container(3, containerCapacity, 0), new Container(4, containerCapacity, 0) };
    addInstance(testContainers[0], "A", 0);
    addInstance(testContainers[0], "A", 1);
    addInstance(testContainers[0], "B", 2);
    addInstance(testContainers[0], "B", 3);
    addInstance(testContainers[1], "A", 4);
    addInstance(testContainers[1], "B", 5);
    addInstance(testContainers[1], "B", 6);
    addInstance(testContainers[2], "A", 7);
    addInstance(testContainers[2], "A", 8);
}