Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 46 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class MesosScheduler method fillResourcesRequirementForBaseContainer.

/**
 * Fill the the resources requirement, i.e. CPU, memory and disk for the given container.
 * This method changes the BaseContainer passed in.
 * <p>
 * Notice: Currently we just make every container homogeneous,
 * requiring maximum resources for every container.
 *
 * @param container the BaseContainer to fill value in
 * @param containerIndex the index of the container
 * @param packing the packing plan
 */
protected void fillResourcesRequirementForBaseContainer(BaseContainer container, Integer containerIndex, PackingPlan packing) {
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    Resource maxResourceContainer = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    double cpu = 0;
    ByteAmount disk = ByteAmount.ZERO;
    ByteAmount mem = ByteAmount.ZERO;
    for (PackingPlan.ContainerPlan cp : packing.getContainers()) {
        Resource containerResource = cp.getRequiredResource();
        cpu = Math.max(cpu, containerResource.getCpu());
        disk = disk.max(containerResource.getDisk());
        mem = mem.max(containerResource.getRam());
    }
    container.cpu = maxResourceContainer.getCpu();
    // Convert them from bytes to MB
    container.diskInMB = maxResourceContainer.getDisk().asMegabytes();
    container.memInMB = maxResourceContainer.getRam().asMegabytes();
    container.ports = SchedulerUtils.ExecutorPort.getRequiredPorts().size();
}

Example 47 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class AuroraSchedulerTest method testOnSchedule.

/**
 * Tests that we can schedule
 */
@Test
public void testOnSchedule() throws Exception {
    AuroraController controller = Mockito.mock(AuroraController.class);
    doReturn(controller).when(scheduler).getController();
    SchedulerStateManagerAdaptor stateManager = mock(SchedulerStateManagerAdaptor.class);
    Config runtime = Mockito.mock(Config.class);
    when(runtime.get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR)).thenReturn(stateManager);
    when(runtime.getStringValue(Key.TOPOLOGY_NAME)).thenReturn(TOPOLOGY_NAME);
    Config mConfig = Mockito.mock(Config.class);
    PowerMockito.mockStatic(Config.class);
    when(Config.toClusterMode(mConfig)).thenReturn(mConfig);
    when(mConfig.getStringValue(eq(AuroraContext.JOB_TEMPLATE), anyString())).thenReturn(AURORA_PATH);
    scheduler.initialize(mConfig, runtime);
    // Fail to schedule due to null PackingPlan
    Assert.assertFalse(scheduler.onSchedule(null));
    PackingPlan plan = new PackingPlan(PACKING_PLAN_ID, new HashSet<PackingPlan.ContainerPlan>());
    assertTrue(plan.getContainers().isEmpty());
    // Fail to schedule due to PackingPlan is empty
    Assert.assertFalse(scheduler.onSchedule(plan));
    // Construct valid PackingPlan
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(PackingTestUtils.testContainerPlan(CONTAINER_ID));
    PackingPlan validPlan = new PackingPlan(PACKING_PLAN_ID, containers);
    // Failed to create job via controller
    doReturn(false).when(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    doReturn(true).when(stateManager).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
    Assert.assertFalse(scheduler.onSchedule(validPlan));
    Mockito.verify(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    Mockito.verify(stateManager).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
    // Happy path
    doReturn(true).when(controller).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    assertTrue(scheduler.onSchedule(validPlan));
    Mockito.verify(controller, Mockito.times(2)).createJob(Matchers.anyMapOf(AuroraField.class, String.class), Matchers.anyMapOf(String.class, String.class));
    Mockito.verify(stateManager, Mockito.times(2)).updatePackingPlan(any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
}

Example 48 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class NomadScheduler method getJobs.

List<Job> getJobs(PackingPlan packing) {
    List<Job> ret = new LinkedList<>();
    PackingPlan homogeneousPackingPlan = getHomogeneousPackingPlan(packing);
    Resource resource = getHomogeneousContainerResource(homogeneousPackingPlan);
    for (int i = 0; i < Runtime.numContainers(this.runtimeConfig); i++) {
        Optional<PackingPlan.ContainerPlan> containerPlan = homogeneousPackingPlan.getContainer(i);
        ret.add(getJob(i, containerPlan, resource));
    }
    return ret;
}

Example 49 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class NomadScheduler method getHomogeneousPackingPlan.

PackingPlan getHomogeneousPackingPlan(PackingPlan packingPlan) {
    // Align resources to maximal requested resource
    PackingPlan updatedPackingPlan = packingPlan.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(this.runtimeConfig), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(this.runtimeConfig));
    return updatedPackingPlan;
}

Example 50 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class AuroraScheduler method onSchedule.

@Override
public boolean onSchedule(PackingPlan packing) {
    if (packing == null || packing.getContainers().isEmpty()) {
        LOG.severe("No container requested. Can't schedule");
        return false;
    }
    LOG.info("Launching topology in aurora");
    // Align the cpu, RAM, disk to the maximal one, and set them to ScheduledResource
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    // Use the ScheduledResource to create aurora properties
    // the ScheduledResource is guaranteed to be set after calling
    // cloneWithHomogeneousScheduledResource in the above code
    Resource containerResource = updatedPackingPlan.getContainers().iterator().next().getScheduledResource().get();
    Map<AuroraField, String> auroraProperties = createAuroraProperties(containerResource);
    Map<String, String> extraProperties = createExtraProperties(containerResource);
    return controller.createJob(auroraProperties, extraProperties);
}