Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 11 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanBuilder method getContainers.

/**
   * Generates the containers that correspond to the current packing plan
   * along with their associated instances.
   *
   * @return Map of containers for the current packing plan, keyed by containerId
   */
@VisibleForTesting
static Map<Integer, Container> getContainers(PackingPlan currentPackingPlan, int paddingPercentage, Map<String, TreeSet<Integer>> componentIndexes, TreeSet<Integer> taskIds) throws ResourceExceededException {
    Map<Integer, Container> containers = new HashMap<>();
    Resource capacity = currentPackingPlan.getMaxContainerResources();
    for (PackingPlan.ContainerPlan currentContainerPlan : currentPackingPlan.getContainers()) {
        Container container = new Container(currentContainerPlan.getId(), capacity, paddingPercentage);
        for (PackingPlan.InstancePlan instancePlan : currentContainerPlan.getInstances()) {
            try {
                addToContainer(container, instancePlan, componentIndexes, taskIds);
            } catch (ResourceExceededException e) {
                throw new ResourceExceededException(String.format("Insufficient container resources to add instancePlan %s to container %s", instancePlan, container), e);
            }
        }
        containers.put(currentContainerPlan.getId(), container);
    }
    return containers;
}

Example 12 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class LaunchRunner method call.

/**
   * Call launcher to launch topology
   *
   * @throws LauncherException
   * @throws PackingException
   * @throws SubmitDryRunResponse
   */
public void call() throws LauncherException, PackingException, SubmitDryRunResponse {
    SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
    TopologyAPI.Topology topology = Runtime.topology(runtime);
    String topologyName = Context.topologyName(config);
    PackingPlan packedPlan = LauncherUtils.getInstance().createPackingPlan(config, runtime);
    if (Context.dryRun(config)) {
        throw new SubmitDryRunResponse(topology, config, packedPlan);
    }
    // initialize the launcher
    launcher.initialize(config, runtime);
    // Set topology def first since we determine whether a topology is running
    // by checking the existence of topology def
    // store the trimmed topology definition into the state manager
    // TODO(rli): log-and-false anti-pattern is too nested on this path. will not refactor
    Boolean result = statemgr.setTopology(trimTopology(topology), topologyName);
    if (result == null || !result) {
        throw new LauncherException(String.format("Failed to set topology definition for topology '%s'", topologyName));
    }
    result = statemgr.setPackingPlan(createPackingPlan(packedPlan), topologyName);
    if (result == null || !result) {
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set packing plan for topology '%s'", topologyName));
    }
    // store the execution state into the state manager
    ExecutionEnvironment.ExecutionState executionState = createExecutionState();
    result = statemgr.setExecutionState(executionState, topologyName);
    if (result == null || !result) {
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set execution state for topology '%s'", topologyName));
    }
    // launch the topology, clear the state if it fails
    if (!launcher.launch(packedPlan)) {
        statemgr.deleteExecutionState(topologyName);
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to launch topology '%s'", topologyName));
    }
}

Example 13 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RuntimeManagerRunner method buildNewPackingPlan.

@VisibleForTesting
PackingPlans.PackingPlan buildNewPackingPlan(PackingPlans.PackingPlan currentProtoPlan, Map<String, Integer> changeRequests, TopologyAPI.Topology topology) throws PackingException {
    PackingPlanProtoDeserializer deserializer = new PackingPlanProtoDeserializer();
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    PackingPlan currentPackingPlan = deserializer.fromProto(currentProtoPlan);
    Map<String, Integer> componentCounts = currentPackingPlan.getComponentCounts();
    Map<String, Integer> componentChanges = parallelismDelta(componentCounts, changeRequests);
    // Create an instance of the packing class
    String repackingClass = Context.repackingClass(config);
    IRepacking packing;
    try {
        // create an instance of the packing class
        packing = ReflectionUtils.newInstance(repackingClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new IllegalArgumentException("Failed to instantiate packing instance: " + repackingClass, e);
    }
    LOG.info("Updating packing plan using " + repackingClass);
    try {
        packing.initialize(config, topology);
        PackingPlan packedPlan = packing.repack(currentPackingPlan, componentChanges);
        return serializer.toProto(packedPlan);
    } finally {
        SysUtils.closeIgnoringExceptions(packing);
    }
}

Example 14 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RuntimeManagerRunner method updateTopologyHandler.

/**
   * Handler to update a topology
   */
@VisibleForTesting
void updateTopologyHandler(String topologyName, String newParallelism) throws TopologyRuntimeManagementException, PackingException, UpdateDryRunResponse {
    LOG.fine(String.format("updateTopologyHandler called for %s with %s", topologyName, newParallelism));
    SchedulerStateManagerAdaptor manager = Runtime.schedulerStateManagerAdaptor(runtime);
    TopologyAPI.Topology topology = manager.getTopology(topologyName);
    Map<String, Integer> changeRequests = parseNewParallelismParam(newParallelism);
    PackingPlans.PackingPlan currentPlan = manager.getPackingPlan(topologyName);
    if (!changeDetected(currentPlan, changeRequests)) {
        throw new TopologyRuntimeManagementException(String.format("The component parallelism request (%s) is the same as the " + "current topology parallelism. Not taking action.", newParallelism));
    }
    PackingPlans.PackingPlan proposedPlan = buildNewPackingPlan(currentPlan, changeRequests, topology);
    if (Context.dryRun(config)) {
        PackingPlanProtoDeserializer deserializer = new PackingPlanProtoDeserializer();
        PackingPlan oldPlan = deserializer.fromProto(currentPlan);
        PackingPlan newPlan = deserializer.fromProto(proposedPlan);
        throw new UpdateDryRunResponse(topology, config, newPlan, oldPlan, changeRequests);
    }
    Scheduler.UpdateTopologyRequest updateTopologyRequest = Scheduler.UpdateTopologyRequest.newBuilder().setCurrentPackingPlan(currentPlan).setProposedPackingPlan(proposedPlan).build();
    LOG.fine("Sending Updating topology request: " + updateTopologyRequest);
    if (!schedulerClient.updateTopology(updateTopologyRequest)) {
        throw new TopologyRuntimeManagementException(String.format("Failed to update topology with Scheduler, updateTopologyRequest=" + updateTopologyRequest));
    }
    // Clean the connection when we are done.
    LOG.fine("Scheduler updated topology successfully.");
}

Example 15 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class ResourceCompliantRRPackingTest method testDefaultResources.

/**
   * Test the scenario where the max container size is the default
   */
@Test
public void testDefaultResources() throws Exception {
    int numContainers = 2;
    PackingPlan packingPlanNoExplicitResourcesConfig = pack(topology);
    Assert.assertEquals(numContainers, packingPlanNoExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanNoExplicitResourcesConfig.getInstanceCount());
}