Examples with ContainerPlan org.apache.heron.spi.packing.PackingPlan.ContainerPlan used on opensource projects

Example 1 with ContainerPlan

use of org.apache.heron.spi.packing.PackingPlan.ContainerPlan in project heron by twitter.

the class HeronMasterDriver method scheduleHeronWorkers.

/**
 * Container allocation is asynchronous. Requests all containers in the input set serially
 * to ensure allocated resources match the required resources.
 */
void scheduleHeronWorkers(Set<ContainerPlan> containers) throws ContainerAllocationException {
    for (ContainerPlan containerPlan : containers) {
        int id = containerPlan.getId();
        if (containerPlans.containsKey(containerPlan.getId())) {
            throw new ContainerAllocationException("Received duplicate allocation request for " + id);
        }
        Resource reqResource = containerPlan.getRequiredResource();
        containerPlans.put(id, containerPlan);
        requestContainerForWorker(id, new HeronWorker(id, reqResource));
    }
}

Example 2 with ContainerPlan

use of org.apache.heron.spi.packing.PackingPlan.ContainerPlan in project heron by twitter.

the class HeronMasterDriver method killWorkers.

/**
 * Terminates any yarn containers associated with the given containers.
 */
public void killWorkers(Set<ContainerPlan> containers) {
    for (ContainerPlan container : containers) {
        LOG.log(Level.INFO, "Find and kill container for worker {0}", container.getId());
        Optional<HeronWorker> worker = multiKeyWorkerMap.lookupByWorkerId(container.getId());
        if (worker.isPresent()) {
            LOG.log(Level.INFO, "Killing container {0} for worker {1}", new Object[] { worker.get().evaluator.getId(), worker.get().workerId });
            AllocatedEvaluator evaluator = multiKeyWorkerMap.detachEvaluatorAndRemove(worker.get());
            evaluator.close();
        } else {
            LOG.log(Level.WARNING, "Did not find worker for {0}", container.getId());
        }
        containerPlans.remove(container.getId());
    }
}

Example 3 with ContainerPlan

use of org.apache.heron.spi.packing.PackingPlan.ContainerPlan in project heron by twitter.

the class LaunchRunnerTest method createRunnerRuntime.

private static Config createRunnerRuntime(org.apache.heron.api.Config topologyConfig) throws Exception {
    Config runtime = spy(Config.newBuilder().build());
    ILauncher launcher = mock(ILauncher.class);
    IPacking packing = mock(IPacking.class);
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    TopologyAPI.Topology topology = createTopology(topologyConfig);
    doReturn(launcher).when(runtime).get(Key.LAUNCHER_CLASS_INSTANCE);
    doReturn(adaptor).when(runtime).get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR);
    doReturn(topology).when(runtime).get(Key.TOPOLOGY_DEFINITION);
    PackingPlan packingPlan = mock(PackingPlan.class);
    when(packingPlan.getContainers()).thenReturn(new HashSet<ContainerPlan>());
    when(packingPlan.getComponentRamDistribution()).thenReturn("ramdist");
    when(packingPlan.getId()).thenReturn("packing_plan_id");
    Set<ContainerPlan> containerPlans = new HashSet<>();
    // just need it to be of size 1
    containerPlans.add(PackingTestUtils.testContainerPlan(1));
    when(packingPlan.getContainers()).thenReturn(containerPlans);
    when(packing.pack()).thenReturn(packingPlan);
    LauncherUtils mockLauncherUtils = mock(LauncherUtils.class);
    when(mockLauncherUtils.createPackingPlan(any(Config.class), any(Config.class))).thenReturn(packingPlan);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    return runtime;
}

Example 4 with ContainerPlan

use of org.apache.heron.spi.packing.PackingPlan.ContainerPlan in project heron by twitter.

the class JsonFormatterUtilsTest method setUp.

@Before
public void setUp() throws Exception {
    final String COMPONENT_A = "exclaim1";
    final String COMPONENT_B = "word";
    ContainerPlan containerPlanA = PackingTestUtils.testContainerPlan(1, new Pair<>(COMPONENT_A, 1), new Pair<>(COMPONENT_A, 3), new Pair<>(COMPONENT_B, 5));
    ContainerPlan containerPlanB = PackingTestUtils.testContainerPlan(2, new Pair<>(COMPONENT_A, 2), new Pair<>(COMPONENT_A, 4), new Pair<>(COMPONENT_B, 6));
    Set<ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlanA);
    containerPlans.add(containerPlanB);
    plan = new PackingPlan("A", containerPlans);
}

Example 5 with ContainerPlan

use of org.apache.heron.spi.packing.PackingPlan.ContainerPlan in project heron by twitter.

the class PackingPlanProvider method getBoltInstanceNames.

public String[] getBoltInstanceNames(String... boltComponents) {
    HashSet<String> boltComponentNames = new HashSet<>();
    Collections.addAll(boltComponentNames, boltComponents);
    PackingPlan packing = get();
    ArrayList<String> boltInstanceNames = new ArrayList<>();
    for (ContainerPlan containerPlan : packing.getContainers()) {
        for (InstancePlan instancePlan : containerPlan.getInstances()) {
            if (!boltComponentNames.contains(instancePlan.getComponentName())) {
                continue;
            }
            String name = "container_" + containerPlan.getId() + "_" + instancePlan.getComponentName() + "_" + instancePlan.getTaskId();
            boltInstanceNames.add(name);
        }
    }
    return boltInstanceNames.toArray(new String[boltInstanceNames.size()]);
}