Examples with InstanceId com.twitter.heron.spi.packing.InstanceId used on opensource projects

Example 21 with InstanceId

use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.

the class CommonPackingTests method testScaleDownOneComponentRemoveContainer.

/**
 * Test the scenario where scaling down removes instances from containers that are most imbalanced
 * (i.e., tending towards homogeneity) first. If there is a tie (e.g. AABB, AB), chooses from the
 * container with the fewest instances, to favor ultimately removing  containers. If there is
 * still a tie, favor removing from higher numbered containers
 */
@Test
public void testScaleDownOneComponentRemoveContainer() throws Exception {
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] initialComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)), new Pair<>(3, new InstanceId(B, 4, 1)), new Pair<>(3, new InstanceId(B, 5, 2)), new Pair<>(4, new InstanceId(B, 6, 3)), new Pair<>(4, new InstanceId(B, 7, 4)) };
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(B, -2);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] expectedComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)), new Pair<>(3, new InstanceId(B, 4, 1)), new Pair<>(3, new InstanceId(B, 5, 2)) };
    doScaleDownTest(initialComponentInstances, componentChanges, expectedComponentInstances);
}

Example 22 with InstanceId

use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.

the class RoundRobinPacking method packInternal.

private PackingPlan packInternal(int numContainer, Map<String, Integer> parallelismMap) {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation(numContainer, parallelismMap);
    // Get the ram map for every instance
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = getInstancesRamMapInContainer(roundRobinAllocation);
    ByteAmount containerDiskInBytes = getContainerDiskHint(roundRobinAllocation);
    double containerCpu = getContainerCpuHint(roundRobinAllocation);
    ByteAmount containerRamHint = getContainerRamHint(roundRobinAllocation);
    LOG.info(String.format("Pack internal: container cpu hint: %f, ram hint: %s, disk hint: %s.", containerCpu, containerDiskInBytes.toString(), containerRamHint.toString()));
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = containerRamPadding;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDiskDefault;
            double instanceCpu = instanceCpuDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
        }
        Resource resource = new Resource(containerCpu, containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    // Check whether it is a valid PackingPlan
    validatePackingPlan(plan);
    return plan;
}

Example 23 with InstanceId

use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.

the class RoundRobinPacking method getInstancesRamMapInContainer.

/**
 * Calculate the ram required by any instance in the container
 *
 * @param allocation the allocation of instances in different container
 * @return A map: (containerId -> (instanceId -> instanceRequiredRam))
 */
private Map<Integer, Map<InstanceId, ByteAmount>> getInstancesRamMapInContainer(Map<Integer, List<InstanceId>> allocation) {
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMapInContainer = new HashMap<>();
    ByteAmount containerRamHint = getContainerRamHint(allocation);
    for (int containerId : allocation.keySet()) {
        List<InstanceId> instanceIds = allocation.get(containerId);
        Map<InstanceId, ByteAmount> ramInsideContainer = new HashMap<>();
        instancesRamMapInContainer.put(containerId, ramInsideContainer);
        List<InstanceId> instancesToBeAccounted = new ArrayList<>();
        // Calculate the actual value
        ByteAmount usedRam = ByteAmount.ZERO;
        for (InstanceId instanceId : instanceIds) {
            String componentName = instanceId.getComponentName();
            if (ramMap.containsKey(componentName)) {
                ByteAmount ram = ramMap.get(componentName);
                ramInsideContainer.put(instanceId, ram);
                usedRam = usedRam.plus(ram);
            } else {
                instancesToBeAccounted.add(instanceId);
            }
        }
        // Now we have calculated ram for instances specified in ComponentRamMap
        // Then to calculate ram for the rest instances
        int instancesToAllocate = instancesToBeAccounted.size();
        if (instancesToAllocate != 0) {
            ByteAmount individualInstanceRam = instanceRamDefault;
            // If container ram is specified
            if (!containerRamHint.equals(NOT_SPECIFIED_NUMBER_VALUE)) {
                // remove ram for heron internal process
                ByteAmount remainingRam = containerRamHint.minus(containerRamPadding).minus(usedRam);
                // Split remaining ram evenly
                individualInstanceRam = remainingRam.divide(instancesToAllocate);
            }
            // Put the results in instancesRam
            for (InstanceId instanceId : instancesToBeAccounted) {
                ramInsideContainer.put(instanceId, individualInstanceRam);
            }
        }
    }
    return instancesRamMapInContainer;
}

Example 24 with InstanceId

use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.

the class AssertPacking method assertPackingPlan.

public static void assertPackingPlan(String expectedTopologyName, Pair<Integer, InstanceId>[] expectedComponentInstances, PackingPlan plan) {
    assertEquals(expectedTopologyName, plan.getId());
    assertEquals("Unexpected number of instances: " + plan.getContainers(), expectedComponentInstances.length, plan.getInstanceCount().intValue());
    // for every instance on a given container...
    Set<Integer> expectedContainerIds = new HashSet<>();
    for (Pair<Integer, InstanceId> expectedComponentInstance : expectedComponentInstances) {
        // verify the expected container exists
        int containerId = expectedComponentInstance.first;
        InstanceId instanceId = expectedComponentInstance.second;
        assertTrue(String.format("Container with id %s not found", containerId), plan.getContainer(containerId).isPresent());
        expectedContainerIds.add(containerId);
        // and that the instance exists on it
        boolean instanceFound = false;
        PackingPlan.ContainerPlan containerPlan = plan.getContainer(containerId).get();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            if (instancePlan.getTaskId() == instanceId.getTaskId()) {
                instanceFound = true;
                assertEquals("Wrong componentName for task " + instancePlan.getTaskId(), instanceId.getComponentName(), instancePlan.getComponentName());
                assertEquals("Wrong getComponentIndex for task " + instancePlan.getTaskId(), instanceId.getComponentIndex(), instancePlan.getComponentIndex());
                break;
            }
        }
        assertTrue(String.format("Container (%s) did not include expected instance with taskId %d", containerPlan, instanceId.getTaskId()), instanceFound);
    }
    Map<Integer, PackingPlan.InstancePlan> taskIds = new HashMap<>();
    Map<String, Set<PackingPlan.InstancePlan>> componentInstances = new HashMap<>();
    for (PackingPlan.ContainerPlan containerPlan : plan.getContainers()) {
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            // check for taskId collisions
            PackingPlan.InstancePlan collisionInstance = taskIds.get(instancePlan.getTaskId());
            assertNull(String.format("Task id collision between instance %s and %s", instancePlan, collisionInstance), collisionInstance);
            taskIds.put(instancePlan.getTaskId(), instancePlan);
            // check for componentIndex collisions
            Set<PackingPlan.InstancePlan> instances = componentInstances.get(instancePlan.getComponentName());
            if (instances != null) {
                for (PackingPlan.InstancePlan instance : instances) {
                    assertTrue(String.format("Component index collision between instance %s and %s", instance, instancePlan), instance.getComponentIndex() != instancePlan.getComponentIndex());
                }
            }
            if (componentInstances.get(instancePlan.getComponentName()) == null) {
                componentInstances.put(instancePlan.getComponentName(), new HashSet<PackingPlan.InstancePlan>());
            }
            componentInstances.get(instancePlan.getComponentName()).add(instancePlan);
        }
    }
    assertEquals(expectedContainerIds.size(), plan.getContainers().size());
}

Example 25 with InstanceId

use of com.twitter.heron.spi.packing.InstanceId in project incubator-heron by apache.

the class PackingPlanBuilderTest method testAddToPackingPlan.

@Test
public void testAddToPackingPlan() throws ResourceExceededException {
    PackingPlan plan = doCreatePackingPlanTest(testContainerInstances);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] added = new Pair[] { new Pair<>(1, new InstanceId("componentB", 4, 1)), new Pair<>(4, new InstanceId("componentA", 5, 2)) };
    PackingPlan updatedPlan = PackingTestHelper.addToTestPackingPlan(TOPOLOGY_ID, plan, PackingTestHelper.toContainerIdComponentNames(added), 0);
    Pair<Integer, InstanceId>[] expected = concat(testContainerInstances, added);
    AssertPacking.assertPackingPlan(TOPOLOGY_ID, expected, updatedPlan);
}