Examples with PackingPlanProtoSerializer com.twitter.heron.spi.packing.PackingPlanProtoSerializer used on opensource projects

Example 6 with PackingPlanProtoSerializer

use of com.twitter.heron.spi.packing.PackingPlanProtoSerializer in project incubator-heron by apache.

the class PackingTestUtils method testProtoPackingPlan.

public static PackingPlans.PackingPlan testProtoPackingPlan(String topologyName, IPacking packing) {
    PackingPlan plan = testPackingPlan(topologyName, packing);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    return serializer.toProto(plan);
}

Example 7 with PackingPlanProtoSerializer

use of com.twitter.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class PackingTestUtils method testProtoPackingPlan.

public static PackingPlans.PackingPlan testProtoPackingPlan(String topologyName, IPacking packing) {
    PackingPlan plan = testPackingPlan(topologyName, packing);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    return serializer.toProto(plan);
}

Example 8 with PackingPlanProtoSerializer

use of com.twitter.heron.spi.packing.PackingPlanProtoSerializer in project incubator-heron by apache.

the class SchedulerUtils method persistUpdatedPackingPlan.

/**
 * Replaces persisted packing plan in state manager.
 */
public static void persistUpdatedPackingPlan(String topologyName, PackingPlan updatedPackingPlan, SchedulerStateManagerAdaptor stateManager) {
    LOG.log(Level.INFO, "Updating scheduled-resource in packing plan: {0}", topologyName);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    if (!stateManager.updatePackingPlan(serializer.toProto(updatedPackingPlan), topologyName)) {
        throw new RuntimeException(String.format("Failed to update packing plan for topology %s", topologyName));
    }
}

Example 9 with PackingPlanProtoSerializer

use of com.twitter.heron.spi.packing.PackingPlanProtoSerializer in project incubator-heron by apache.

the class UpdateTopologyManager method updateTopology.

private void updateTopology(final PackingPlans.PackingPlan existingProtoPackingPlan, final PackingPlans.PackingPlan proposedProtoPackingPlan, SchedulerStateManagerAdaptor stateManager) throws ExecutionException, InterruptedException {
    String topologyName = Runtime.topologyName(runtime);
    PackingPlan existingPackingPlan = deserializer.fromProto(existingProtoPackingPlan);
    PackingPlan proposedPackingPlan = deserializer.fromProto(proposedProtoPackingPlan);
    Preconditions.checkArgument(proposedPackingPlan.getContainers().size() > 0, String.format("proposed packing plan must have at least 1 container %s", proposedPackingPlan));
    ContainerDelta containerDelta = new ContainerDelta(existingPackingPlan.getContainers(), proposedPackingPlan.getContainers());
    int newContainerCount = containerDelta.getContainersToAdd().size();
    int removableContainerCount = containerDelta.getContainersToRemove().size();
    String message = String.format("Topology change requires %s new containers and removing %s " + "existing containers, but the scheduler does not support scaling, aborting. " + "Existing packing plan: %s, proposed packing plan: %s", newContainerCount, removableContainerCount, existingPackingPlan, proposedPackingPlan);
    Preconditions.checkState(newContainerCount + removableContainerCount == 0 || scalableScheduler.isPresent(), message);
    TopologyAPI.Topology topology = getTopology(stateManager, topologyName);
    boolean initiallyRunning = topology.getState() == TopologyAPI.TopologyState.RUNNING;
    // deactivate and sleep
    if (initiallyRunning) {
        // Update the topology since the state should have changed from RUNNING to PAUSED
        // Will throw exceptions internally if tmaster fails to deactivate
        deactivateTopology(stateManager, topology, proposedPackingPlan);
    }
    Set<PackingPlan.ContainerPlan> updatedContainers = new HashSet<>(proposedPackingPlan.getContainers());
    // to state manager quickly, otherwise the scheduler might penalize for thrashing on start-up
    if (newContainerCount > 0 && scalableScheduler.isPresent()) {
        Set<PackingPlan.ContainerPlan> containersToAdd = containerDelta.getContainersToAdd();
        Set<PackingPlan.ContainerPlan> containersAdded = scalableScheduler.get().addContainers(containersToAdd);
        // Update the PackingPlan with new container-ids
        if (containersAdded != null) {
            if (containersAdded.size() != containersToAdd.size()) {
                throw new RuntimeException("Scheduler failed to add requested containers. Requested " + containersToAdd.size() + ", added " + containersAdded.size() + ". " + "The topology can be in a strange stage. " + "Please check carefully or redeploy the topology !!");
            }
            updatedContainers.removeAll(containersToAdd);
            updatedContainers.addAll(containersAdded);
        }
    }
    PackingPlan updatedPackingPlan = new PackingPlan(proposedPackingPlan.getId(), updatedContainers);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    PackingPlans.PackingPlan updatedProtoPackingPlan = serializer.toProto(updatedPackingPlan);
    LOG.fine("The updated Packing Plan: " + updatedProtoPackingPlan);
    // update packing plan to trigger the scaling event
    logInfo("Update new PackingPlan: %s", stateManager.updatePackingPlan(updatedProtoPackingPlan, topologyName));
    // reactivate topology
    if (initiallyRunning) {
        // wait before reactivating to give the tmaster a chance to receive the packing update and
        // delete the packing plan. Instead we could message tmaster to invalidate the physical plan
        // and/or possibly even update the packing plan directly
        SysUtils.sleep(Duration.ofSeconds(10));
        // Will throw exceptions internally if tmaster fails to deactivate
        reactivateTopology(stateManager, topology, removableContainerCount);
    }
    if (removableContainerCount > 0 && scalableScheduler.isPresent()) {
        scalableScheduler.get().removeContainers(containerDelta.getContainersToRemove());
    }
}