Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 36 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class HeronMasterDriverTest method scheduleHeronWorkersRequestsContainersForPacking.

@Test
public void scheduleHeronWorkersRequestsContainersForPacking() throws Exception {
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    PackingPlan.ContainerPlan container1 = PackingTestUtils.testContainerPlan(1, 1, 2);
    containers.add(container1);
    PackingPlan.ContainerPlan container2 = PackingTestUtils.testContainerPlan(2, 1, 2, 3);
    containers.add(container2);
    PackingPlan packing = new PackingPlan("packingId", containers);
    spyDriver.scheduleHeronWorkers(packing);
    verify(mockRequestor, times(2)).submit(any(EvaluatorRequest.class));
    verify(spyDriver, times(1)).requestContainerForWorker(eq(1), anyHeronWorker());
    verify(spyDriver, times(1)).requestContainerForWorker(eq(2), anyHeronWorker());
    verify(spyDriver, times(1)).createEvaluatorRequest(getCpu(container1), getRam(container1));
    verify(spyDriver, times(1)).createEvaluatorRequest(getCpu(container2), getRam(container2));
}

Example 37 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RoundRobinPacking method pack.

@Override
public PackingPlan pack() {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation();
    // Get the ram map for every instance
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = getInstancesRamMapInContainer(roundRobinAllocation);
    ByteAmount containerDiskInBytes = getContainerDiskHint(roundRobinAllocation);
    double containerCpu = getContainerCpuHint(roundRobinAllocation);
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = DEFAULT_RAM_PADDING_PER_CONTAINER;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDiskDefault;
            double instanceCpu = instanceCpuDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
        }
        Resource resource = new Resource(containerCpu, containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    // Check whether it is a valid PackingPlan
    validatePackingPlan(plan);
    return plan;
}

Example 38 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class AssertPacking method assertPackingPlan.

public static void assertPackingPlan(String expectedTopologyName, Pair<Integer, InstanceId>[] expectedComponentInstances, PackingPlan plan) {
    assertEquals(expectedTopologyName, plan.getId());
    assertEquals("Unexpected number of instances: " + plan.getContainers(), expectedComponentInstances.length, plan.getInstanceCount().intValue());
    // for every instance on a given container...
    Set<Integer> expectedContainerIds = new HashSet<>();
    for (Pair<Integer, InstanceId> expectedComponentInstance : expectedComponentInstances) {
        // verify the expected container exists
        int containerId = expectedComponentInstance.first;
        InstanceId instanceId = expectedComponentInstance.second;
        assertTrue(String.format("Container with id %s not found", containerId), plan.getContainer(containerId).isPresent());
        expectedContainerIds.add(containerId);
        // and that the instance exists on it
        boolean instanceFound = false;
        PackingPlan.ContainerPlan containerPlan = plan.getContainer(containerId).get();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            if (instancePlan.getTaskId() == instanceId.getTaskId()) {
                instanceFound = true;
                assertEquals("Wrong componentName for task " + instancePlan.getTaskId(), instanceId.getComponentName(), instancePlan.getComponentName());
                assertEquals("Wrong getComponentIndex for task " + instancePlan.getTaskId(), instanceId.getComponentIndex(), instancePlan.getComponentIndex());
                break;
            }
        }
        assertTrue(String.format("Container (%s) did not include expected instance with taskId %d", containerPlan, instanceId.getTaskId()), instanceFound);
    }
    Map<Integer, PackingPlan.InstancePlan> taskIds = new HashMap<>();
    Map<String, Set<PackingPlan.InstancePlan>> componentInstances = new HashMap<>();
    for (PackingPlan.ContainerPlan containerPlan : plan.getContainers()) {
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            // check for taskId collisions
            PackingPlan.InstancePlan collisionInstance = taskIds.get(instancePlan.getTaskId());
            assertNull(String.format("Task id collision between instance %s and %s", instancePlan, collisionInstance), collisionInstance);
            taskIds.put(instancePlan.getTaskId(), instancePlan);
            // check for componentIndex collisions
            Set<PackingPlan.InstancePlan> instances = componentInstances.get(instancePlan.getComponentName());
            if (instances != null) {
                for (PackingPlan.InstancePlan instance : instances) {
                    assertTrue(String.format("Component index collision between instance %s and %s", instance, instancePlan), instance.getComponentIndex() != instancePlan.getComponentIndex());
                }
            }
            if (componentInstances.get(instancePlan.getComponentName()) == null) {
                componentInstances.put(instancePlan.getComponentName(), new HashSet<PackingPlan.InstancePlan>());
            }
            componentInstances.get(instancePlan.getComponentName()).add(instancePlan);
        }
    }
    assertEquals(expectedContainerIds.size(), plan.getContainers().size());
}

Example 39 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
   * Test the scenario where container level resource config are set
   */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerMaxRamHint(containerRam);
    topologyConfig.setContainerMaxDiskHint(containerDisk);
    topologyConfig.setContainerMaxCpuHint(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = pack(topologyExplicitResourcesConfig);
    Assert.assertEquals(1, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), SPOUT_NAME, 4);
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(Math.round(PackingUtils.increaseBy(totalInstances * instanceDefaultResources.getCpu(), DEFAULT_CONTAINER_PADDING)), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 40 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class FirstFitDecreasingPackingTest method testPartialRamMap.

/**
   * Test the scenario ram map config is partially set
   */
@Test
public void testPartialRamMap() throws Exception {
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(2, packingPlanExplicitRamMap.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, instanceDefaultResources.getRam(), maxContainerRam);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}