Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 16 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class LocalSchedulerTest method testRemoveContainer.

/**
 * Verify containers can be removed by Local Scheduler
 */
@Test
public void testRemoveContainer() throws Exception {
    final int LOCAL_NUM_CONTAINER = 6;
    // verify plan is deployed and containers are created
    Mockito.doNothing().when(scheduler).startExecutorMonitor(Mockito.anyInt(), Mockito.any(Process.class), Mockito.anySet());
    Process[] processes = new Process[LOCAL_NUM_CONTAINER];
    Set<PackingPlan.ContainerPlan> existingContainers = new HashSet<>();
    for (int i = 0; i < LOCAL_NUM_CONTAINER; i++) {
        processes[i] = Mockito.mock(Process.class);
        Set<PackingPlan.InstancePlan> instances = (i == 0) ? null : PackingTestUtils.testContainerPlan(i).getInstances();
        Mockito.doReturn(processes[i]).when(scheduler).startExecutorProcess(i, instances);
        if (i > 0) {
            // ignore the container for TManager. existing containers simulate the containers created
            // by packing plan
            existingContainers.add(PackingTestUtils.testContainerPlan(i));
        }
    }
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    Mockito.when(packingPlan.getContainers()).thenReturn(existingContainers);
    Assert.assertTrue(scheduler.onSchedule(packingPlan));
    verifyIdsOfLaunchedContainers(0, 1, 2, 3, 4, 5);
    Mockito.verify(scheduler, Mockito.times(LOCAL_NUM_CONTAINER)).startExecutor(Mockito.anyInt(), Mockito.anySet());
    Set<PackingPlan.ContainerPlan> containersToRemove = new HashSet<>();
    PackingPlan.ContainerPlan containerToRemove = PackingTestUtils.testContainerPlan(LOCAL_NUM_CONTAINER - 1);
    containersToRemove.add(containerToRemove);
    scheduler.removeContainers(containersToRemove);
    verifyIdsOfLaunchedContainers(0, 1, 2, 3, 4);
    Mockito.verify(processes[LOCAL_NUM_CONTAINER - 1]).destroy();
    // verify no new process restarts
    Mockito.verify(scheduler, Mockito.times(LOCAL_NUM_CONTAINER)).startExecutor(Mockito.anyInt(), Mockito.anySet());
    containersToRemove.clear();
    containersToRemove.add(PackingTestUtils.testContainerPlan(1));
    containersToRemove.add(PackingTestUtils.testContainerPlan(2));
    scheduler.removeContainers(containersToRemove);
    verifyIdsOfLaunchedContainers(0, 3, 4);
    Mockito.verify(processes[1]).destroy();
    Mockito.verify(processes[2]).destroy();
    // verify no new process restarts
    Mockito.verify(scheduler, Mockito.times(LOCAL_NUM_CONTAINER)).startExecutor(Mockito.anyInt(), Mockito.anySet());
}

Example 17 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class KubernetesController method getContainerResource.

Resource getContainerResource(PackingPlan packingPlan) {
    // Align resources to maximal requested resource
    PackingPlan updatedPackingPlan = packingPlan.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtimeConfiguration), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtimeConfiguration));
    return updatedPackingPlan.getContainers().iterator().next().getScheduledResource().get();
}

Example 18 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class MarathonScheduler method getTopologyConf.

protected String getTopologyConf(PackingPlan packing) {
    config = Config.newBuilder().putAll(config).put(Key.TOPOLOGY_BINARY_FILE, Context.topologyBinaryFile(config)).build();
    ObjectMapper mapper = new ObjectMapper();
    // TODO (nlu): use heterogeneous resources
    // Align resources to maximal requested resource
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    Resource containerResource = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    // Create app conf list for each container
    ArrayNode instances = mapper.createArrayNode();
    for (int i = 0; i < Runtime.numContainers(runtime); i++) {
        ObjectNode instance = mapper.createObjectNode();
        instance.put(MarathonConstants.ID, marathonGroupId + "/" + Integer.toString(i));
        instance.put(MarathonConstants.COMMAND, getExecutorCommand(i));
        instance.put(MarathonConstants.CPU, containerResource.getCpu());
        instance.set(MarathonConstants.CONTAINER, getContainer(mapper));
        instance.put(MarathonConstants.MEMORY, containerResource.getRam().asMegabytes());
        instance.put(MarathonConstants.DISK, containerResource.getDisk().asMegabytes());
        instance.put(MarathonConstants.INSTANCES, 1);
        instance.set(MarathonConstants.LABELS, getLabels(mapper));
        instance.set(MarathonConstants.FETCH, getFetchList(mapper));
        instances.add(instance);
    }
    // Create marathon group for a topology
    ObjectNode appConf = mapper.createObjectNode();
    appConf.put(MarathonConstants.ID, marathonGroupId);
    appConf.set(MarathonConstants.APPS, instances);
    return appConf.toString();
}

Example 19 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class SchedulerMainTest method updateNumContainersIfNeeded.

@Test
public void updateNumContainersIfNeeded() {
    int configuredNumContainers = 4;
    int configuredNumStreamManagers = configuredNumContainers - 1;
    int packingPlanSize = 1;
    SubmitterMain submitterMain = new SubmitterMain(Config.newBuilder().put(Key.PACKING_CLASS, "org.apache.heron.packing.roundrobin.ResourceCompliantRRPacking").build(), null);
    PackingPlan packingPlan = PackingTestUtils.testPackingPlan(TOPOLOGY_NAME, new RoundRobinPacking());
    assertEquals(packingPlanSize, packingPlan.getContainers().size());
    org.apache.heron.api.Config apiConfig = new org.apache.heron.api.Config();
    apiConfig.setNumStmgrs(configuredNumStreamManagers);
    TopologyAPI.Topology initialTopology = TopologyTests.createTopology(TOPOLOGY_NAME, apiConfig, new HashMap<String, Integer>(), new HashMap<String, Integer>());
    Config initialConfig = Config.newBuilder().put(Key.NUM_CONTAINERS, configuredNumContainers).put(Key.PACKING_CLASS, RoundRobinPacking.class.getName()).put(Key.TOPOLOGY_DEFINITION, initialTopology).build();
    // assert preconditions
    assertEquals(Integer.toString(configuredNumStreamManagers), apiConfig.get(org.apache.heron.api.Config.TOPOLOGY_STMGRS));
    assertEquals(configuredNumStreamManagers, TopologyUtils.getNumContainers(initialTopology));
    assertContainerCount(configuredNumStreamManagers, initialConfig);
    Config newConfig = submitterMain.updateNumContainersIfNeeded(initialConfig, initialTopology, packingPlan);
    assertContainerCount(packingPlanSize, newConfig);
}

Example 20 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class LaunchRunnerTest method createRunnerRuntime.

private static Config createRunnerRuntime(org.apache.heron.api.Config topologyConfig) throws Exception {
    Config runtime = spy(Config.newBuilder().build());
    ILauncher launcher = mock(ILauncher.class);
    IPacking packing = mock(IPacking.class);
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    TopologyAPI.Topology topology = createTopology(topologyConfig);
    doReturn(launcher).when(runtime).get(Key.LAUNCHER_CLASS_INSTANCE);
    doReturn(adaptor).when(runtime).get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR);
    doReturn(topology).when(runtime).get(Key.TOPOLOGY_DEFINITION);
    PackingPlan packingPlan = mock(PackingPlan.class);
    when(packingPlan.getContainers()).thenReturn(new HashSet<ContainerPlan>());
    when(packingPlan.getComponentRamDistribution()).thenReturn("ramdist");
    when(packingPlan.getId()).thenReturn("packing_plan_id");
    Set<ContainerPlan> containerPlans = new HashSet<>();
    // just need it to be of size 1
    containerPlans.add(PackingTestUtils.testContainerPlan(1));
    when(packingPlan.getContainers()).thenReturn(containerPlans);
    when(packing.pack()).thenReturn(packingPlan);
    LauncherUtils mockLauncherUtils = mock(LauncherUtils.class);
    when(mockLauncherUtils.createPackingPlan(any(Config.class), any(Config.class))).thenReturn(packingPlan);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    return runtime;
}