Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 1 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class LaunchRunnerTest method createRunnerRuntime.

private static Config createRunnerRuntime(com.twitter.heron.api.Config topologyConfig) throws Exception {
    Config runtime = spy(Config.newBuilder().build());
    ILauncher launcher = mock(ILauncher.class);
    IPacking packing = mock(IPacking.class);
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    TopologyAPI.Topology topology = createTopology(topologyConfig);
    doReturn(launcher).when(runtime).get(Key.LAUNCHER_CLASS_INSTANCE);
    doReturn(adaptor).when(runtime).get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR);
    doReturn(topology).when(runtime).get(Key.TOPOLOGY_DEFINITION);
    PackingPlan packingPlan = mock(PackingPlan.class);
    when(packingPlan.getContainers()).thenReturn(new HashSet<ContainerPlan>());
    when(packingPlan.getComponentRamDistribution()).thenReturn("ramdist");
    when(packingPlan.getId()).thenReturn("packing_plan_id");
    Set<ContainerPlan> containerPlans = new HashSet<>();
    // just need it to be of size 1
    containerPlans.add(PackingTestUtils.testContainerPlan(1));
    when(packingPlan.getContainers()).thenReturn(containerPlans);
    when(packing.pack()).thenReturn(packingPlan);
    LauncherUtils mockLauncherUtils = mock(LauncherUtils.class);
    when(mockLauncherUtils.createPackingPlan(any(Config.class), any(Config.class))).thenReturn(packingPlan);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    return runtime;
}

Example 2 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class AuroraScheduler method onSchedule.

@Override
public boolean onSchedule(PackingPlan packing) {
    if (packing == null || packing.getContainers().isEmpty()) {
        LOG.severe("No container requested. Can't schedule");
        return false;
    }
    LOG.info("Launching topology in aurora");
    // Align the cpu, ram, disk to the maximal one
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    Resource containerResource = packing.getContainers().iterator().next().getRequiredResource();
    Map<AuroraField, String> auroraProperties = createAuroraProperties(containerResource);
    return controller.createJob(auroraProperties);
}

Example 3 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class UpdateTopologyManager method updateTopology.

private void updateTopology(final PackingPlans.PackingPlan existingProtoPackingPlan, final PackingPlans.PackingPlan proposedProtoPackingPlan, SchedulerStateManagerAdaptor stateManager) throws ExecutionException, InterruptedException {
    String topologyName = Runtime.topologyName(runtime);
    PackingPlan existingPackingPlan = deserializer.fromProto(existingProtoPackingPlan);
    PackingPlan proposedPackingPlan = deserializer.fromProto(proposedProtoPackingPlan);
    Preconditions.checkArgument(proposedPackingPlan.getContainers().size() > 0, String.format("proposed packing plan must have at least 1 container %s", proposedPackingPlan));
    ContainerDelta containerDelta = new ContainerDelta(existingPackingPlan.getContainers(), proposedPackingPlan.getContainers());
    int newContainerCount = containerDelta.getContainersToAdd().size();
    int removableContainerCount = containerDelta.getContainersToRemove().size();
    String message = String.format("Topology change requires %s new containers and removing %s " + "existing containers, but the scheduler does not support scaling, aborting. " + "Existing packing plan: %s, proposed packing plan: %s", newContainerCount, removableContainerCount, existingPackingPlan, proposedPackingPlan);
    Preconditions.checkState(newContainerCount + removableContainerCount == 0 || scalableScheduler.isPresent(), message);
    TopologyAPI.Topology topology = stateManager.getTopology(topologyName);
    boolean initiallyRunning = topology.getState() == TopologyAPI.TopologyState.RUNNING;
    // fetch the topology, which will need to be updated
    TopologyAPI.Topology updatedTopology = getUpdatedTopology(topologyName, proposedPackingPlan, stateManager);
    // deactivate and sleep
    if (initiallyRunning) {
        deactivateTopology(stateManager, updatedTopology);
    }
    // to state manager quickly, otherwise the scheduler might penalize for thrashing on start-up
    if (newContainerCount > 0 && scalableScheduler.isPresent()) {
        scalableScheduler.get().addContainers(containerDelta.getContainersToAdd());
    }
    // update parallelism in updatedTopology since TMaster checks that
    // Sum(parallelism) == Sum(instances)
    logFine("Update new Topology: %s", stateManager.updateTopology(updatedTopology, topologyName));
    // update packing plan to trigger the scaling event
    logFine("Update new PackingPlan: %s", stateManager.updatePackingPlan(proposedProtoPackingPlan, topologyName));
    // delete the physical plan so TMaster doesn't try to re-establish it on start-up.
    logFine("Deleted Physical Plan: %s", stateManager.deletePhysicalPlan(topologyName));
    // reactivate topology
    if (initiallyRunning) {
        reactivateTopology(stateManager, updatedTopology, removableContainerCount);
    }
    if (removableContainerCount > 0 && scalableScheduler.isPresent()) {
        scalableScheduler.get().removeContainers(containerDelta.getContainersToRemove());
    }
}

Example 4 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SubmitDryRunRenderTest method setUp.

@Before
public void setUp() throws Exception {
    final String COMPONENT_A = "exclaim1";
    final String COMPONENT_B = "word";
    ContainerPlan containerPlanA = PackingTestUtils.testContainerPlan(1, new Pair<>(COMPONENT_A, 1), new Pair<>(COMPONENT_A, 3), new Pair<>(COMPONENT_B, 5));
    ContainerPlan containerPlanB = PackingTestUtils.testContainerPlan(2, new Pair<>(COMPONENT_A, 2), new Pair<>(COMPONENT_A, 4), new Pair<>(COMPONENT_B, 6));
    Set<ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlanA);
    containerPlans.add(containerPlanB);
    plan = new PackingPlan("A", containerPlans);
}

Example 5 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class LauncherUtilsTest method generatesPackingPlan.

@Test
public void generatesPackingPlan() throws Exception {
    final String PACKING_CLASS = "nonExistingTestPackingClass";
    final PackingPlan mockPackingPlan = Mockito.mock(PackingPlan.class);
    IPacking mockPacking = Mockito.mock(IPacking.class);
    Mockito.when(mockPacking.pack()).thenReturn(mockPackingPlan);
    PowerMockito.spy(ReflectionUtils.class);
    PowerMockito.doReturn(mockPacking).when(ReflectionUtils.class, "newInstance", PACKING_CLASS);
    TopologyAPI.Topology mockTopology = PowerMockito.mock(TopologyAPI.Topology.class);
    Config mockConfig = Mockito.mock(Config.class);
    Mockito.when(mockConfig.getStringValue(Key.PACKING_CLASS)).thenReturn(PACKING_CLASS);
    Mockito.when(mockConfig.get(Key.TOPOLOGY_DEFINITION)).thenReturn(mockTopology);
    PackingPlan resultPacking = LauncherUtils.getInstance().createPackingPlan(mockConfig, mockConfig);
    Assert.assertEquals(mockPackingPlan, resultPacking);
    Mockito.verify(mockPacking).initialize(Mockito.any(Config.class), Mockito.eq(mockTopology));
    Mockito.verify(mockPacking).pack();
    Mockito.verify(mockPacking).close();
}