Examples with Resource com.twitter.heron.spi.packing.Resource used on opensource projects

Example 21 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPacking method initialize.

@Override
public void initialize(Config config, TopologyAPI.Topology inputTopology) {
    this.topology = inputTopology;
    this.numContainers = TopologyUtils.getNumContainers(topology);
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    resetToFirstContainer();
}

Example 22 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPacking method pack.

@Override
public PackingPlan pack() {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation();
    // Get the ram map for every instance
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = getInstancesRamMapInContainer(roundRobinAllocation);
    ByteAmount containerDiskInBytes = getContainerDiskHint(roundRobinAllocation);
    double containerCpu = getContainerCpuHint(roundRobinAllocation);
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = DEFAULT_RAM_PADDING_PER_CONTAINER;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDiskDefault;
            double instanceCpu = instanceCpuDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
        }
        Resource resource = new Resource(containerCpu, containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    // Check whether it is a valid PackingPlan
    validatePackingPlan(plan);
    return plan;
}

Example 23 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtils method computeTotalResourceChange.

/**
   * Identifies the resources reclaimed by the components that will be scaled down
   *
   * @return Total resources reclaimed
   */
public static Resource computeTotalResourceChange(TopologyAPI.Topology topology, Map<String, Integer> componentChanges, Resource defaultInstanceResources, ScalingDirection scalingDirection) {
    double cpu = 0;
    ByteAmount ram = ByteAmount.ZERO;
    ByteAmount disk = ByteAmount.ZERO;
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<String, Integer> componentsToScale = PackingUtils.getComponentsToScale(componentChanges, scalingDirection);
    for (String component : componentsToScale.keySet()) {
        int parallelismChange = Math.abs(componentChanges.get(component));
        cpu += parallelismChange * defaultInstanceResources.getCpu();
        disk = disk.plus(defaultInstanceResources.getDisk().multiply(parallelismChange));
        if (ramMap.containsKey(component)) {
            ram = ram.plus(ramMap.get(component).multiply(parallelismChange));
        } else {
            ram = ram.plus(defaultInstanceResources.getRam().multiply(parallelismChange));
        }
    }
    return new Resource(cpu, ram, disk);
}

Example 24 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
   * Test the scenario where container level resource config are set
   */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerMaxRamHint(containerRam);
    topologyConfig.setContainerMaxDiskHint(containerDisk);
    topologyConfig.setContainerMaxCpuHint(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = pack(topologyExplicitResourcesConfig);
    Assert.assertEquals(1, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), SPOUT_NAME, 4);
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(Math.round(PackingUtils.increaseBy(totalInstances * instanceDefaultResources.getCpu(), DEFAULT_CONTAINER_PADDING)), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 25 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilderTest method generatePacking.

private static PackingPlan generatePacking(Map<Integer, List<InstanceId>> basePacking) throws RuntimeException {
    Resource resource = new Resource(2.0, ByteAmount.fromGigabytes(6), ByteAmount.fromGigabytes(25));
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : basePacking.keySet()) {
        List<InstanceId> instanceList = basePacking.get(containerId);
        Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
        for (InstanceId instanceId : instanceList) {
            String componentName = instanceId.getComponentName();
            Resource instanceResource;
            switch(componentName) {
                case "bolt":
                    instanceResource = new Resource(1.0, ByteAmount.fromGigabytes(2), ByteAmount.fromGigabytes(10));
                    break;
                case "spout":
                    instanceResource = new Resource(1.0, ByteAmount.fromGigabytes(3), ByteAmount.fromGigabytes(10));
                    break;
                default:
                    throw new RuntimeException(String.format("%s is not a valid component name", componentName));
            }
            instancePlans.add(new PackingPlan.InstancePlan(instanceId, instanceResource));
        }
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
        containerPlans.add(containerPlan);
    }
    return new PackingPlan("", containerPlans);
}