Examples with PackingPlanProtoSerializer org.apache.heron.spi.packing.PackingPlanProtoSerializer used on opensource projects

Example 1 with PackingPlanProtoSerializer

use of org.apache.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class PackingTestUtils method testProtoPackingPlan.

public static PackingPlans.PackingPlan testProtoPackingPlan(String topologyName, IPacking packing) {
    PackingPlan plan = testPackingPlan(topologyName, packing);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    return serializer.toProto(plan);
}

Example 2 with PackingPlanProtoSerializer

use of org.apache.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class UpdateTopologyManager method updateTopology.

private void updateTopology(final PackingPlans.PackingPlan existingProtoPackingPlan, final PackingPlans.PackingPlan proposedProtoPackingPlan, SchedulerStateManagerAdaptor stateManager) throws ExecutionException, InterruptedException {
    String topologyName = Runtime.topologyName(runtime);
    PackingPlan existingPackingPlan = deserializer.fromProto(existingProtoPackingPlan);
    PackingPlan proposedPackingPlan = deserializer.fromProto(proposedProtoPackingPlan);
    Preconditions.checkArgument(proposedPackingPlan.getContainers().size() > 0, String.format("proposed packing plan must have at least 1 container %s", proposedPackingPlan));
    ContainerDelta containerDelta = new ContainerDelta(existingPackingPlan.getContainers(), proposedPackingPlan.getContainers());
    int newContainerCount = containerDelta.getContainersToAdd().size();
    int removableContainerCount = containerDelta.getContainersToRemove().size();
    String message = String.format("Topology change requires %s new containers and removing %s " + "existing containers, but the scheduler does not support scaling, aborting. " + "Existing packing plan: %s, proposed packing plan: %s", newContainerCount, removableContainerCount, existingPackingPlan, proposedPackingPlan);
    Preconditions.checkState(newContainerCount + removableContainerCount == 0 || scalableScheduler.isPresent(), message);
    TopologyAPI.Topology topology = getTopology(stateManager, topologyName);
    boolean initiallyRunning = topology.getState() == TopologyAPI.TopologyState.RUNNING;
    // deactivate and sleep
    if (initiallyRunning) {
        // Update the topology since the state should have changed from RUNNING to PAUSED
        // Will throw exceptions internally if tmanager fails to deactivate
        deactivateTopology(stateManager, topology, proposedPackingPlan);
    }
    Set<PackingPlan.ContainerPlan> updatedContainers = new HashSet<>(proposedPackingPlan.getContainers());
    // to state manager quickly, otherwise the scheduler might penalize for thrashing on start-up
    if (newContainerCount > 0 && scalableScheduler.isPresent()) {
        Set<PackingPlan.ContainerPlan> containersToAdd = containerDelta.getContainersToAdd();
        Set<PackingPlan.ContainerPlan> containersAdded = scalableScheduler.get().addContainers(containersToAdd);
        // Update the PackingPlan with new container-ids
        if (containersAdded != null) {
            if (containersAdded.size() != containersToAdd.size()) {
                throw new RuntimeException("Scheduler failed to add requested containers. Requested " + containersToAdd.size() + ", added " + containersAdded.size() + ". " + "The topology can be in a strange stage. " + "Please check carefully or redeploy the topology !!");
            }
            updatedContainers.removeAll(containersToAdd);
            updatedContainers.addAll(containersAdded);
        }
    }
    PackingPlan updatedPackingPlan = new PackingPlan(proposedPackingPlan.getId(), updatedContainers);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    PackingPlans.PackingPlan updatedProtoPackingPlan = serializer.toProto(updatedPackingPlan);
    LOG.fine("The updated Packing Plan: " + updatedProtoPackingPlan);
    // update packing plan to trigger the scaling event
    logInfo("Update new PackingPlan: %s", stateManager.updatePackingPlan(updatedProtoPackingPlan, topologyName));
    // reactivate topology
    if (initiallyRunning) {
        // wait before reactivating to give the tmanager a chance to receive the packing update and
        // delete the packing plan. Instead we could message tmanager to invalidate the physical plan
        // and/or possibly even update the packing plan directly
        SysUtils.sleep(Duration.ofSeconds(10));
        // Will throw exceptions internally if tmanager fails to deactivate
        reactivateTopology(stateManager, topology, removableContainerCount);
    }
    if (removableContainerCount > 0 && scalableScheduler.isPresent()) {
        scalableScheduler.get().removeContainers(containerDelta.getContainersToRemove());
    }
}

Example 3 with PackingPlanProtoSerializer

use of org.apache.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class SchedulerUtils method persistUpdatedPackingPlan.

/**
 * Replaces persisted packing plan in state manager.
 */
public static void persistUpdatedPackingPlan(String topologyName, PackingPlan updatedPackingPlan, SchedulerStateManagerAdaptor stateManager) {
    LOG.log(Level.INFO, "Updating scheduled-resource in packing plan: {0}", topologyName);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    if (!stateManager.updatePackingPlan(serializer.toProto(updatedPackingPlan), topologyName)) {
        throw new RuntimeException(String.format("Failed to update packing plan for topology %s", topologyName));
    }
}

Example 4 with PackingPlanProtoSerializer

use of org.apache.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class RuntimeManagerRunner method buildNewPackingPlan.

@VisibleForTesting
PackingPlans.PackingPlan buildNewPackingPlan(PackingPlans.PackingPlan currentProtoPlan, Map<String, Integer> changeRequests, Integer containerNum, TopologyAPI.Topology topology) throws PackingException {
    PackingPlanProtoDeserializer deserializer = new PackingPlanProtoDeserializer();
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    PackingPlan currentPackingPlan = deserializer.fromProto(currentProtoPlan);
    Map<String, Integer> componentCounts = currentPackingPlan.getComponentCounts();
    Map<String, Integer> componentChanges = parallelismDelta(componentCounts, changeRequests);
    // Create an instance of the packing class
    String repackingClass = Context.repackingClass(config);
    IRepacking packing;
    try {
        // create an instance of the packing class
        packing = ReflectionUtils.newInstance(repackingClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new IllegalArgumentException("Failed to instantiate packing instance: " + repackingClass, e);
    }
    LOG.info("Updating packing plan using " + repackingClass);
    try {
        packing.initialize(config, topology);
        PackingPlan packedPlan = null;
        if (containerNum == null) {
            packedPlan = packing.repack(currentPackingPlan, componentChanges);
        } else {
            packedPlan = packing.repack(currentPackingPlan, containerNum, componentChanges);
        }
        return serializer.toProto(packedPlan);
    } finally {
        SysUtils.closeIgnoringExceptions(packing);
    }
}

Example 5 with PackingPlanProtoSerializer

use of org.apache.heron.spi.packing.PackingPlanProtoSerializer in project heron by twitter.

the class UpdateTopologyManagerTest method init.

@Before
public void init() {
    Integer[] instanceIndexA = new Integer[] { 37, 48, 59 };
    Integer[] instanceIndexB = new Integer[] { 17, 22 };
    currentContainerPlan = buildContainerSet(new Integer[] { 1, 2, 3, 4 }, instanceIndexA);
    proposedContainerPlan = buildContainerSet(new Integer[] { 1, 3, 5, 6 }, instanceIndexB);
    expectedContainersToAdd = buildContainerSet(new Integer[] { 5, 6 }, instanceIndexB);
    expectedContainersToRemove = buildContainerSet(new Integer[] { 2, 4 }, instanceIndexA);
    PackingPlanProtoSerializer serializer = new PackingPlanProtoSerializer();
    PackingPlan currentPacking = new PackingPlan("current", currentContainerPlan);
    proposedPacking = new PackingPlan("proposed", proposedContainerPlan);
    currentProtoPlan = serializer.toProto(currentPacking);
    proposedProtoPlan = serializer.toProto(proposedPacking);
    testTopology = TopologyTests.createTopology(TOPOLOGY_NAME, new org.apache.heron.api.Config(), "spoutname", "boltname", 1, 1);
    assertEquals(TopologyAPI.TopologyState.RUNNING, testTopology.getState());
}