Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 76 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SubmitterMain method submitTopology.

/**
   * Submit a topology
   * 1. Instantiate necessary resources
   * 2. Valid whether it is legal to submit a topology
   * 3. Call LauncherRunner
   *
   */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 77 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RoundRobinPackingTest method testDefaultResources.

@Test
public void testDefaultResources() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // No explicit resources required
    TopologyAPI.Topology topologyNoExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanNoExplicitResourcesConfig = getRoundRobinPackingPlan(topologyNoExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanNoExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanNoExplicitResourcesConfig.getInstanceCount());
}

Example 78 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class RuntimeManagerRunner method changeDetected.

private static boolean changeDetected(PackingPlans.PackingPlan currentProtoPlan, Map<String, Integer> changeRequests) {
    PackingPlanProtoDeserializer deserializer = new PackingPlanProtoDeserializer();
    PackingPlan currentPlan = deserializer.fromProto(currentProtoPlan);
    for (String component : changeRequests.keySet()) {
        if (changeRequests.get(component) != currentPlan.getComponentCounts().get(component)) {
            return true;
        }
    }
    return false;
}

Example 79 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SchedulerMainTest method updateNumContainersIfNeeded.

@Test
public void updateNumContainersIfNeeded() {
    int configuredNumContainers = 4;
    int configuredNumStreamManagers = configuredNumContainers - 1;
    int packingPlanSize = 1;
    SubmitterMain submitterMain = new SubmitterMain(null, null);
    PackingPlan packingPlan = PackingTestUtils.testPackingPlan(TOPOLOGY_NAME, new RoundRobinPacking());
    assertEquals(packingPlanSize, packingPlan.getContainers().size());
    com.twitter.heron.api.Config apiConfig = new com.twitter.heron.api.Config();
    apiConfig.setNumStmgrs(configuredNumStreamManagers);
    TopologyAPI.Topology initialTopology = TopologyTests.createTopology(TOPOLOGY_NAME, apiConfig, new HashMap<String, Integer>(), new HashMap<String, Integer>());
    Config initialConfig = Config.newBuilder().put(Key.NUM_CONTAINERS, configuredNumContainers).put(Key.PACKING_CLASS, RoundRobinPacking.class.getName()).put(Key.TOPOLOGY_DEFINITION, initialTopology).build();
    // assert preconditions
    assertEquals(Integer.toString(configuredNumStreamManagers), apiConfig.get(com.twitter.heron.api.Config.TOPOLOGY_STMGRS));
    assertEquals(configuredNumStreamManagers, TopologyUtils.getNumContainers(initialTopology));
    assertContainerCount(configuredNumStreamManagers, initialConfig);
    Config newConfig = submitterMain.updateNumContainersIfNeeded(initialConfig, initialTopology, packingPlan);
    assertContainerCount(packingPlanSize, newConfig);
}

Example 80 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SchedulerMainTest method getTestPacking.

private SettableFuture<PackingPlans.PackingPlan> getTestPacking() {
    PackingPlans.PackingPlan packingPlan = PackingTestUtils.testProtoPackingPlan("testTopology", new RoundRobinPacking());
    final SettableFuture<PackingPlans.PackingPlan> future = SettableFuture.create();
    future.set(packingPlan);
    return future;
}